From 5b0a2e8a6e4289e9044ed342697d60db769ec568 Mon Sep 17 00:00:00 2001 From: Eric Promislow Date: Wed, 25 Sep 2024 10:59:51 -0700 Subject: [PATCH] make charts --- ...r-monitoring-crd-104.1.2-rc.1+up57.0.3.tgz | Bin 0 -> 299417 bytes ...ncher-monitoring-104.1.2-rc.1+up57.0.3.tgz | Bin 0 -> 479998 bytes .../104.1.2-rc.1+up57.0.3/Chart.yaml | 10 + .../104.1.2-rc.1+up57.0.3/README.md | 24 + .../files/crd-manifest.tgz | Bin 0 -> 308599 bytes .../templates/_helpers.tpl | 30 + .../104.1.2-rc.1+up57.0.3/templates/jobs.yaml | 102 + .../templates/manifest.yaml | 8 + .../104.1.2-rc.1+up57.0.3/templates/rbac.yaml | 76 + .../templates/validate-psp-install.yaml | 7 + .../104.1.2-rc.1+up57.0.3/values.yaml | 17 + .../104.1.2-rc.1+up57.0.3/.editorconfig | 5 + .../104.1.2-rc.1+up57.0.3/.helmignore | 29 + .../104.1.2-rc.1+up57.0.3/CHANGELOG.md | 47 + .../104.1.2-rc.1+up57.0.3/CONTRIBUTING.md | 12 + .../104.1.2-rc.1+up57.0.3/Chart.yaml | 126 + .../104.1.2-rc.1+up57.0.3/README.md | 1080 ++++ .../104.1.2-rc.1+up57.0.3/app-README.md | 46 + .../charts/grafana/.helmignore | 23 + .../charts/grafana/Chart.yaml | 39 + .../charts/grafana/README.md | 770 +++ .../grafana/dashboards/custom-dashboard.json | 1 + .../charts/grafana/templates/NOTES.txt | 55 + .../charts/grafana/templates/_config.tpl | 171 + .../charts/grafana/templates/_helpers.tpl | 305 + .../charts/grafana/templates/_pod.tpl | 1296 ++++ .../charts/grafana/templates/clusterrole.yaml | 25 + .../grafana/templates/clusterrolebinding.yaml | 24 + .../grafana/templates/configSecret.yaml | 43 + .../configmap-dashboard-provider.yaml | 15 + .../charts/grafana/templates/configmap.yaml | 15 + .../templates/dashboards-json-configmap.yaml | 38 + .../charts/grafana/templates/deployment.yaml | 53 + .../grafana/templates/extra-manifests.yaml | 4 + .../grafana/templates/headless-service.yaml | 22 + .../charts/grafana/templates/hpa.yaml | 52 + .../templates/image-renderer-deployment.yaml | 131 + .../grafana/templates/image-renderer-hpa.yaml | 47 + .../image-renderer-network-policy.yaml | 79 + .../templates/image-renderer-service.yaml | 31 + .../image-renderer-servicemonitor.yaml | 48 + .../charts/grafana/templates/ingress.yaml | 78 + .../grafana/templates/networkpolicy.yaml | 61 + .../grafana/templates/nginx-config.yaml | 94 + .../templates/poddisruptionbudget.yaml | 22 + .../grafana/templates/podsecuritypolicy.yaml | 45 + .../charts/grafana/templates/pvc.yaml | 41 + .../charts/grafana/templates/role.yaml | 32 + .../charts/grafana/templates/rolebinding.yaml | 25 + .../charts/grafana/templates/secret-env.yaml | 14 + .../charts/grafana/templates/secret.yaml | 16 + .../charts/grafana/templates/service.yaml | 61 + .../grafana/templates/serviceaccount.yaml | 17 + .../grafana/templates/servicemonitor.yaml | 68 + .../charts/grafana/templates/statefulset.yaml | 58 + .../templates/tests/test-configmap.yaml | 20 + .../tests/test-podsecuritypolicy.yaml | 32 + .../grafana/templates/tests/test-role.yaml | 17 + .../templates/tests/test-rolebinding.yaml | 20 + .../templates/tests/test-serviceaccount.yaml | 12 + .../charts/grafana/templates/tests/test.yaml | 53 + .../charts/grafana/values.yaml | 1315 ++++ .../charts/hardenedKubelet/.helmignore | 23 + .../charts/hardenedKubelet/Chart.yaml | 15 + .../charts/hardenedKubelet/README.md | 90 + .../hardenedKubelet/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/hardenedKubelet/values.yaml | 166 + .../charts/hardenedNodeExporter/.helmignore | 23 + .../charts/hardenedNodeExporter/Chart.yaml | 15 + .../charts/hardenedNodeExporter/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/hardenedNodeExporter/values.yaml | 166 + .../charts/k3sServer/.helmignore | 23 + .../charts/k3sServer/Chart.yaml | 15 + .../charts/k3sServer/README.md | 90 + .../charts/k3sServer/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../k3sServer/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../k3sServer/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/k3sServer/values.yaml | 166 + .../charts/kube-state-metrics/.helmignore | 21 + .../charts/kube-state-metrics/Chart.yaml | 32 + .../charts/kube-state-metrics/README.md | 85 + .../kube-state-metrics/templates/NOTES.txt | 23 + .../kube-state-metrics/templates/_helpers.tpl | 196 + .../templates/ciliumnetworkpolicy.yaml | 33 + .../templates/clusterrolebinding.yaml | 20 + .../templates/crs-configmap.yaml | 16 + .../templates/deployment.yaml | 314 + .../templates/extra-manifests.yaml | 4 + .../templates/kubeconfig-secret.yaml | 12 + .../templates/networkpolicy.yaml | 43 + .../kube-state-metrics/templates/pdb.yaml | 18 + .../templates/podsecuritypolicy.yaml | 39 + .../templates/psp-clusterrole.yaml | 19 + .../templates/psp-clusterrolebinding.yaml | 16 + .../templates/rbac-configmap.yaml | 22 + .../kube-state-metrics/templates/role.yaml | 215 + .../templates/rolebinding.yaml | 24 + .../kube-state-metrics/templates/service.yaml | 49 + .../templates/serviceaccount.yaml | 17 + .../templates/servicemonitor.yaml | 126 + .../templates/stsdiscovery-role.yaml | 26 + .../templates/stsdiscovery-rolebinding.yaml | 17 + .../templates/verticalpodautoscaler.yaml | 44 + .../charts/kube-state-metrics/values.yaml | 491 ++ .../kubeAdmControllerManager/.helmignore | 23 + .../kubeAdmControllerManager/Chart.yaml | 15 + .../charts/kubeAdmControllerManager/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../kubeAdmControllerManager/values.yaml | 166 + .../charts/kubeAdmEtcd/.helmignore | 23 + .../charts/kubeAdmEtcd/Chart.yaml | 15 + .../charts/kubeAdmEtcd/README.md | 90 + .../charts/kubeAdmEtcd/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../kubeAdmEtcd/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/kubeAdmEtcd/values.yaml | 166 + .../charts/kubeAdmProxy/.helmignore | 23 + .../charts/kubeAdmProxy/Chart.yaml | 15 + .../charts/kubeAdmProxy/README.md | 90 + .../kubeAdmProxy/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/kubeAdmProxy/values.yaml | 166 + .../charts/kubeAdmScheduler/.helmignore | 23 + .../charts/kubeAdmScheduler/Chart.yaml | 15 + .../charts/kubeAdmScheduler/README.md | 90 + .../kubeAdmScheduler/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/kubeAdmScheduler/values.yaml | 166 + .../charts/prometheus-adapter/.helmignore | 21 + .../charts/prometheus-adapter/Chart.yaml | 28 + .../charts/prometheus-adapter/README.md | 160 + .../prometheus-adapter/templates/NOTES.txt | 9 + .../prometheus-adapter/templates/_helpers.tpl | 113 + .../templates/certmanager.yaml | 76 + .../cluster-role-binding-auth-delegator.yaml | 20 + .../cluster-role-binding-resource-reader.yaml | 20 + .../cluster-role-resource-reader.yaml | 24 + .../templates/configmap.yaml | 97 + .../templates/custom-metrics-apiservice.yaml | 34 + ...stom-metrics-cluster-role-binding-hpa.yaml | 24 + .../custom-metrics-cluster-role.yaml | 17 + .../templates/deployment.yaml | 143 + .../external-metrics-apiservice.yaml | 34 + ...rnal-metrics-cluster-role-binding-hpa.yaml | 20 + .../external-metrics-cluster-role.yaml | 21 + .../prometheus-adapter/templates/pdb.yaml | 23 + .../prometheus-adapter/templates/psp.yaml | 66 + .../resource-metrics-apiservice.yaml | 34 + ...resource-metrics-cluster-role-binding.yaml | 20 + .../resource-metrics-cluster-role.yaml | 23 + .../templates/role-binding-auth-reader.yaml | 21 + .../prometheus-adapter/templates/secret.yaml | 17 + .../prometheus-adapter/templates/service.yaml | 27 + .../templates/serviceaccount.yaml | 18 + .../charts/prometheus-adapter/values.yaml | 277 + .../prometheus-node-exporter/.helmignore | 21 + .../prometheus-node-exporter/Chart.yaml | 25 + .../charts/prometheus-node-exporter/README.md | 97 + .../templates/NOTES.txt | 29 + .../templates/_helpers.tpl | 236 + .../templates/clusterrole.yaml | 19 + .../templates/clusterrolebinding.yaml | 20 + .../templates/daemonset.yaml | 309 + .../templates/endpoints.yaml | 18 + .../templates/extra-manifests.yaml | 4 + .../templates/networkpolicy.yaml | 23 + .../templates/podmonitor.yaml | 91 + .../templates/psp-clusterrole.yaml | 14 + .../templates/psp-clusterrolebinding.yaml | 16 + .../templates/psp.yaml | 49 + .../templates/rbac-configmap.yaml | 16 + .../templates/service.yaml | 29 + .../templates/serviceaccount.yaml | 17 + .../templates/servicemonitor.yaml | 71 + .../templates/verticalpodautoscaler.yaml | 40 + .../prometheus-node-exporter/values.yaml | 530 ++ .../charts/rke2ControllerManager/.helmignore | 23 + .../charts/rke2ControllerManager/Chart.yaml | 15 + .../charts/rke2ControllerManager/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2ControllerManager/values.yaml | 166 + .../charts/rke2Etcd/.helmignore | 23 + .../charts/rke2Etcd/Chart.yaml | 15 + .../charts/rke2Etcd/README.md | 90 + .../charts/rke2Etcd/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rke2Etcd/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rke2Etcd/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2Etcd/values.yaml | 166 + .../charts/rke2IngressNginx/.helmignore | 23 + .../charts/rke2IngressNginx/Chart.yaml | 15 + .../charts/rke2IngressNginx/README.md | 90 + .../rke2IngressNginx/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2IngressNginx/values.yaml | 166 + .../charts/rke2Proxy/.helmignore | 23 + .../charts/rke2Proxy/Chart.yaml | 15 + .../charts/rke2Proxy/README.md | 90 + .../charts/rke2Proxy/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rke2Proxy/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rke2Proxy/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2Proxy/values.yaml | 166 + .../charts/rke2Scheduler/.helmignore | 23 + .../charts/rke2Scheduler/Chart.yaml | 15 + .../charts/rke2Scheduler/README.md | 90 + .../rke2Scheduler/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2Scheduler/values.yaml | 166 + .../charts/rkeControllerManager/.helmignore | 23 + .../charts/rkeControllerManager/Chart.yaml | 15 + .../charts/rkeControllerManager/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeControllerManager/values.yaml | 166 + .../charts/rkeEtcd/.helmignore | 23 + .../charts/rkeEtcd/Chart.yaml | 15 + .../charts/rkeEtcd/README.md | 90 + .../charts/rkeEtcd/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rkeEtcd/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rkeEtcd/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeEtcd/values.yaml | 166 + .../charts/rkeIngressNginx/.helmignore | 23 + .../charts/rkeIngressNginx/Chart.yaml | 15 + .../charts/rkeIngressNginx/README.md | 90 + .../rkeIngressNginx/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeIngressNginx/values.yaml | 166 + .../charts/rkeProxy/.helmignore | 23 + .../charts/rkeProxy/Chart.yaml | 15 + .../charts/rkeProxy/README.md | 90 + .../charts/rkeProxy/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rkeProxy/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rkeProxy/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeProxy/values.yaml | 166 + .../charts/rkeScheduler/.helmignore | 23 + .../charts/rkeScheduler/Chart.yaml | 15 + .../charts/rkeScheduler/README.md | 90 + .../rkeScheduler/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeScheduler/values.yaml | 166 + .../charts/windowsExporter/.helmignore | 21 + .../charts/windowsExporter/Chart.yaml | 17 + .../charts/windowsExporter/README.md | 42 + .../scripts/configure-firewall.ps1 | 31 + .../windowsExporter/templates/_helpers.tpl | 216 + .../windowsExporter/templates/config.yaml | 14 + .../windowsExporter/templates/daemonset.yaml | 200 + .../windowsExporter/templates/podmonitor.yaml | 91 + .../templates/scriptConfig.yaml | 14 + .../windowsExporter/templates/service.yaml | 32 + .../templates/serviceaccount.yaml | 17 + .../templates/servicemonitor.yaml | 75 + .../charts/windowsExporter/values.yaml | 366 ++ .../files/ingress-nginx/nginx.json | 1445 +++++ .../request-handling-performance.json | 963 +++ .../cluster/rancher-cluster-nodes.json | 793 +++ .../rancher/cluster/rancher-cluster.json | 776 +++ .../files/rancher/fleet/bundle.json | 246 + .../files/rancher/fleet/bundledeployment.json | 219 + .../files/rancher/fleet/cluster.json | 484 ++ .../files/rancher/fleet/clustergroup.json | 468 ++ .../rancher/fleet/controller-runtime.json | 454 ++ .../files/rancher/fleet/gitrepo.json | 325 + .../rancher/home/rancher-default-home.json | 1290 ++++ .../files/rancher/k8s/rancher-etcd-nodes.json | 687 +++ .../files/rancher/k8s/rancher-etcd.json | 669 ++ .../k8s/rancher-k8s-components-nodes.json | 527 ++ .../rancher/k8s/rancher-k8s-components.json | 519 ++ .../rancher/nodes/rancher-node-detail.json | 805 +++ .../files/rancher/nodes/rancher-node.json | 792 +++ .../performance/performance-debugging.json | 1652 +++++ .../rancher/pods/rancher-pod-containers.json | 636 ++ .../files/rancher/pods/rancher-pod.json | 636 ++ .../workloads/rancher-workload-pods.json | 652 ++ .../rancher/workloads/rancher-workload.json | 652 ++ .../delete-workloads-with-old-labels.sh | 14 + .../104.1.2-rc.1+up57.0.3/templates/NOTES.txt | 4 + .../templates/_helpers.tpl | 459 ++ .../templates/alertmanager/alertmanager.yaml | 191 + .../templates/alertmanager/extrasecret.yaml | 20 + .../templates/alertmanager/ingress.yaml | 78 + .../alertmanager/ingressperreplica.yaml | 67 + .../alertmanager/podDisruptionBudget.yaml | 21 + .../templates/alertmanager/psp-role.yaml | 23 + .../alertmanager/psp-rolebinding.yaml | 20 + .../templates/alertmanager/psp.yaml | 47 + .../templates/alertmanager/secret.yaml | 35 + .../templates/alertmanager/service.yaml | 68 + .../alertmanager/serviceaccount.yaml | 21 + .../alertmanager/servicemonitor.yaml | 84 + .../alertmanager/serviceperreplica.yaml | 49 + .../templates/exporters/core-dns/service.yaml | 24 + .../exporters/core-dns/servicemonitor.yaml | 58 + .../kube-api-server/servicemonitor.yaml | 57 + .../kube-controller-manager/endpoints.yaml | 22 + .../kube-controller-manager/service.yaml | 29 + .../servicemonitor.yaml | 69 + .../templates/exporters/kube-dns/service.yaml | 28 + .../exporters/kube-dns/servicemonitor.yaml | 71 + .../exporters/kube-etcd/endpoints.yaml | 20 + .../exporters/kube-etcd/service.yaml | 27 + .../exporters/kube-etcd/servicemonitor.yaml | 75 + .../exporters/kube-proxy/endpoints.yaml | 20 + .../exporters/kube-proxy/service.yaml | 27 + .../exporters/kube-proxy/servicemonitor.yaml | 63 + .../exporters/kube-scheduler/endpoints.yaml | 22 + .../exporters/kube-scheduler/service.yaml | 29 + .../kube-scheduler/servicemonitor.yaml | 69 + .../kube-state-metrics/validate.yaml | 7 + .../exporters/kubelet/servicemonitor.yaml | 246 + .../exporters/node-exporter/validate.yaml | 3 + .../templates/extra-objects.yaml | 4 + .../grafana/configmap-dashboards.yaml | 24 + .../grafana/configmaps-datasources.yaml | 81 + .../alertmanager-overview.yaml | 616 ++ .../grafana/dashboards-1.14/apiserver.yaml | 1772 ++++++ .../dashboards-1.14/cluster-total.yaml | 1882 ++++++ .../dashboards-1.14/controller-manager.yaml | 1196 ++++ .../grafana/dashboards-1.14/etcd.yaml | 1229 ++++ .../dashboards-1.14/grafana-overview.yaml | 635 ++ .../grafana/dashboards-1.14/k8s-coredns.yaml | 1534 +++++ .../k8s-resources-cluster.yaml | 3088 ++++++++++ .../k8s-resources-multicluster.yaml | 24 + .../k8s-resources-namespace.yaml | 2797 +++++++++ .../dashboards-1.14/k8s-resources-node.yaml | 1026 ++++ .../dashboards-1.14/k8s-resources-pod.yaml | 2469 ++++++++ .../k8s-resources-windows-cluster.yaml | 24 + .../k8s-resources-windows-namespace.yaml | 24 + .../k8s-resources-windows-pod.yaml | 24 + .../k8s-resources-workload.yaml | 2024 ++++++ .../k8s-resources-workloads-namespace.yaml | 2189 +++++++ .../k8s-windows-cluster-rsrc-use.yaml | 24 + .../k8s-windows-node-rsrc-use.yaml | 24 + .../grafana/dashboards-1.14/kubelet.yaml | 2256 +++++++ .../dashboards-1.14/namespace-by-pod.yaml | 1464 +++++ .../namespace-by-workload.yaml | 1736 ++++++ .../node-cluster-rsrc-use.yaml | 1063 ++++ .../dashboards-1.14/node-rsrc-use.yaml | 1089 ++++ .../grafana/dashboards-1.14/nodes-darwin.yaml | 1073 ++++ .../grafana/dashboards-1.14/nodes.yaml | 1066 ++++ .../persistentvolumesusage.yaml | 587 ++ .../grafana/dashboards-1.14/pod-total.yaml | 1228 ++++ .../prometheus-remote-write.yaml | 1674 +++++ .../grafana/dashboards-1.14/prometheus.yaml | 1235 ++++ .../grafana/dashboards-1.14/proxy.yaml | 1276 ++++ .../grafana/dashboards-1.14/scheduler.yaml | 1118 ++++ .../dashboards-1.14/workload-total.yaml | 1438 +++++ .../templates/grafana/namespaces.yaml | 13 + .../_prometheus-operator.tpl | 7 + .../_prometheus-operator-webhook.tpl | 6 + .../deployment/deployment.yaml | 143 + .../admission-webhooks/deployment/pdb.yaml | 15 + .../deployment/service.yaml | 58 + .../deployment/serviceaccount.yaml | 15 + .../ciliumnetworkpolicy-createSecret.yaml | 36 + .../ciliumnetworkpolicy-patchWebhook.yaml | 36 + .../job-patch/clusterrole.yaml | 33 + .../job-patch/clusterrolebinding.yaml | 20 + .../job-patch/job-createSecret.yaml | 73 + .../job-patch/job-patchWebhook.yaml | 74 + .../job-patch/networkpolicy-createSecret.yaml | 33 + .../job-patch/networkpolicy-patchWebhook.yaml | 33 + .../admission-webhooks/job-patch/psp.yaml | 47 + .../admission-webhooks/job-patch/role.yaml | 21 + .../job-patch/rolebinding.yaml | 21 + .../job-patch/serviceaccount.yaml | 17 + .../mutatingWebhookConfiguration.yaml | 77 + .../validatingWebhookConfiguration.yaml | 77 + .../prometheus-operator/certmanager.yaml | 55 + .../ciliumnetworkpolicy.yaml | 40 + .../prometheus-operator/clusterrole.yaml | 109 + .../clusterrolebinding.yaml | 16 + .../prometheus-operator/deployment.yaml | 204 + .../prometheus-operator/networkpolicy.yaml | 29 + .../prometheus-operator/psp-clusterrole.yaml | 21 + .../psp-clusterrolebinding.yaml | 18 + .../templates/prometheus-operator/psp.yaml | 46 + .../prometheus-operator/service.yaml | 57 + .../prometheus-operator/serviceaccount.yaml | 14 + .../prometheus-operator/servicemonitor.yaml | 57 + .../verticalpodautoscaler.yaml | 40 + .../templates/prometheus/_rules.tpl | 44 + .../additionalAlertRelabelConfigs.yaml | 16 + .../additionalAlertmanagerConfigs.yaml | 16 + .../prometheus/additionalPrometheusRules.yaml | 43 + .../prometheus/additionalScrapeConfigs.yaml | 20 + .../prometheus/ciliumnetworkpolicy.yaml | 27 + .../templates/prometheus/clusterrole.yaml | 30 + .../prometheus/clusterrolebinding.yaml | 18 + .../templates/prometheus/csi-secret.yaml | 12 + .../templates/prometheus/extrasecret.yaml | 20 + .../templates/prometheus/ingress.yaml | 77 + .../prometheus/ingressThanosSidecar.yaml | 77 + .../prometheus/ingressperreplica.yaml | 67 + .../templates/prometheus/networkpolicy.yaml | 34 + .../templates/prometheus/nginx-config.yaml | 68 + .../prometheus/podDisruptionBudget.yaml | 25 + .../templates/prometheus/podmonitors.yaml | 38 + .../templates/prometheus/prometheus.yaml | 472 ++ .../templates/prometheus/psp-clusterrole.yaml | 22 + .../prometheus/psp-clusterrolebinding.yaml | 19 + .../templates/prometheus/psp.yaml | 58 + .../rules-1.14/alertmanager.rules.yaml | 305 + .../rules-1.14/config-reloaders.yaml | 57 + .../templates/prometheus/rules-1.14/etcd.yaml | 461 ++ .../prometheus/rules-1.14/general.rules.yaml | 125 + ...les.container_cpu_usage_seconds_total.yaml | 43 + .../k8s.rules.container_memory_cache.yaml | 42 + .../k8s.rules.container_memory_rss.yaml | 42 + .../k8s.rules.container_memory_swap.yaml | 42 + ...es.container_memory_working_set_bytes.yaml | 42 + .../k8s.rules.container_resource.yaml | 168 + .../rules-1.14/k8s.rules.pod_owner.yaml | 107 + .../prometheus/rules-1.14/k8s.rules.yaml | 237 + .../kube-apiserver-availability.rules.yaml | 273 + .../kube-apiserver-burnrate.rules.yaml | 440 ++ .../kube-apiserver-histogram.rules.yaml | 53 + .../rules-1.14/kube-apiserver-slos.yaml | 159 + .../kube-prometheus-general.rules.yaml | 49 + .../kube-prometheus-node-recording.rules.yaml | 93 + .../rules-1.14/kube-scheduler.rules.yaml | 135 + .../rules-1.14/kube-state-metrics.yaml | 152 + .../prometheus/rules-1.14/kubelet.rules.yaml | 65 + .../rules-1.14/kubernetes-apps.yaml | 568 ++ .../rules-1.14/kubernetes-resources.yaml | 282 + .../rules-1.14/kubernetes-storage.yaml | 216 + .../kubernetes-system-apiserver.yaml | 193 + .../kubernetes-system-controller-manager.yaml | 55 + .../kubernetes-system-kube-proxy.yaml | 56 + .../rules-1.14/kubernetes-system-kubelet.yaml | 379 ++ .../kubernetes-system-scheduler.yaml | 54 + .../rules-1.14/kubernetes-system.yaml | 87 + .../rules-1.14/node-exporter.rules.yaml | 188 + .../prometheus/rules-1.14/node-exporter.yaml | 801 +++ .../prometheus/rules-1.14/node-network.yaml | 55 + .../prometheus/rules-1.14/node.rules.yaml | 109 + .../rules-1.14/prometheus-operator.yaml | 253 + .../prometheus/rules-1.14/prometheus.yaml | 707 +++ .../rules-1.14/windows.node.rules.yaml | 301 + .../rules-1.14/windows.pod.rules.yaml | 158 + .../templates/prometheus/secret.yaml | 15 + .../templates/prometheus/service.yaml | 80 + .../prometheus/serviceThanosSidecar.yaml | 39 + .../serviceThanosSidecarExternal.yaml | 46 + .../templates/prometheus/serviceaccount.yaml | 21 + .../templates/prometheus/servicemonitor.yaml | 97 + .../servicemonitorThanosSidecar.yaml | 55 + .../templates/prometheus/servicemonitors.yaml | 47 + .../prometheus/serviceperreplica.yaml | 54 + .../rancher-monitoring/clusterrole.yaml | 135 + .../rancher-monitoring/config-role.yaml | 48 + .../rancher-monitoring/dashboard-role.yaml | 47 + .../addons/ingress-nginx-dashboard.yaml | 18 + .../rancher/cluster-dashboards.yaml | 17 + .../dashboards/rancher/default-dashboard.yaml | 17 + .../dashboards/rancher/fleet-dashboards.yaml | 17 + .../dashboards/rancher/k8s-dashboards.yaml | 31 + .../dashboards/rancher/nodes-dashboards.yaml | 17 + .../rancher/performance-dashboards.yaml | 18 + .../dashboards/rancher/pods-dashboards.yaml | 17 + .../rancher/workload-dashboards.yaml | 17 + .../exporters/fleet/servicemonitor.yaml | 53 + .../exporters/ingress-nginx/service.yaml | 27 + .../ingress-nginx/servicemonitor.yaml | 49 + .../exporters/rancher/servicemonitor.yaml | 58 + .../rancher-monitoring/hardened.yaml | 147 + .../rancher-monitoring/upgrade/configmap.yaml | 13 + .../rancher-monitoring/upgrade/job.yaml | 46 + .../rancher-monitoring/upgrade/rbac.yaml | 131 + .../templates/thanos-ruler/extrasecret.yaml | 20 + .../templates/thanos-ruler/ingress.yaml | 77 + .../thanos-ruler/podDisruptionBudget.yaml | 21 + .../templates/thanos-ruler/ruler.yaml | 189 + .../templates/thanos-ruler/secret.yaml | 26 + .../templates/thanos-ruler/service.yaml | 53 + .../thanos-ruler/serviceaccount.yaml | 20 + .../thanos-ruler/servicemonitor.yaml | 82 + .../templates/validate-install-crd.yaml | 23 + .../templates/validate-psp-install.yaml | 7 + .../104.1.2-rc.1+up57.0.3/values.yaml | 5431 +++++++++++++++++ index.yaml | 144 + 581 files changed, 105235 insertions(+) create mode 100644 assets/rancher-monitoring-crd/rancher-monitoring-crd-104.1.2-rc.1+up57.0.3.tgz create mode 100644 assets/rancher-monitoring/rancher-monitoring-104.1.2-rc.1+up57.0.3.tgz create mode 100644 charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/Chart.yaml create mode 100644 charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/README.md create mode 100644 charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/files/crd-manifest.tgz create mode 100644 charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/jobs.yaml create mode 100644 charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/manifest.yaml create mode 100644 charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/rbac.yaml create mode 100644 charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/.editorconfig create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/CHANGELOG.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/CONTRIBUTING.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/app-README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/dashboards/custom-dashboard.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/_config.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/_pod.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/configSecret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/configmap-dashboard-provider.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/dashboards-json-configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/deployment.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/extra-manifests.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/headless-service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/hpa.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-deployment.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-hpa.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-network-policy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/ingress.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/nginx-config.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/poddisruptionbudget.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/podsecuritypolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/pvc.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/rolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/secret-env.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/secret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/statefulset.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-rolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/crs-configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/deployment.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/extra-manifests.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/kubeconfig-secret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/pdb.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/podsecuritypolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/rbac-configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/rolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/certmanager.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/deployment.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/pdb.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/psp.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/secret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/daemonset.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/endpoints.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/extra-manifests.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/podmonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/psp.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/rbac-configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/.helmignore create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/Chart.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/README.md create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/scripts/configure-firewall.ps1 create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/config.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/daemonset.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/podmonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/scriptConfig.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/values.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/ingress-nginx/nginx.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/ingress-nginx/request-handling-performance.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/cluster/rancher-cluster-nodes.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/cluster/rancher-cluster.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/bundle.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/bundledeployment.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/cluster.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/clustergroup.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/controller-runtime.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/gitrepo.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/home/rancher-default-home.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-etcd-nodes.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-etcd.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-k8s-components-nodes.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-k8s-components.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/nodes/rancher-node-detail.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/nodes/rancher-node.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/performance/performance-debugging.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/pods/rancher-pod-containers.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/pods/rancher-pod.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/workloads/rancher-workload-pods.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/workloads/rancher-workload.json create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/upgrade/scripts/delete-workloads-with-old-labels.sh create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/alertmanager.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/extrasecret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/ingress.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/ingressperreplica.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/podDisruptionBudget.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/psp-role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/psp-rolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/psp.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/secret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/serviceperreplica.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/core-dns/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/core-dns/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-api-server/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-controller-manager/endpoints.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-controller-manager/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-controller-manager/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-dns/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-dns/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-etcd/endpoints.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-etcd/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-etcd/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-proxy/endpoints.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-proxy/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-proxy/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-scheduler/endpoints.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-scheduler/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-scheduler/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-state-metrics/validate.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kubelet/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/node-exporter/validate.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/extra-objects.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/configmap-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/configmaps-datasources.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/apiserver.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/cluster-total.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/controller-manager.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/etcd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/grafana-overview.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-coredns.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/kubelet.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/node-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/nodes-darwin.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/nodes.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/pod-total.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/prometheus.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/scheduler.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/workload-total.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/namespaces.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/_prometheus-operator.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/certmanager.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/ciliumnetworkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/deployment.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/psp.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/verticalpodautoscaler.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/_rules.tpl create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalAlertRelabelConfigs.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalAlertmanagerConfigs.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalPrometheusRules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalScrapeConfigs.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ciliumnetworkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/csi-secret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/extrasecret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ingress.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ingressThanosSidecar.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ingressperreplica.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/nginx-config.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/podDisruptionBudget.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/podmonitors.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/prometheus.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/psp.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/config-reloaders.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/etcd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/general.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-state-metrics.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubelet.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-resources.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node-exporter.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node-exporter.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node-network.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/prometheus-operator.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/prometheus.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/windows.node.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/windows.pod.rules.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/secret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceThanosSidecar.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceThanosSidecarExternal.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/servicemonitorThanosSidecar.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/servicemonitors.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceperreplica.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/clusterrole.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/config-role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboard-role.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/hardened.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/upgrade/configmap.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/upgrade/job.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/upgrade/rbac.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/extrasecret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/ingress.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/podDisruptionBudget.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/ruler.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/secret.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/service.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/values.yaml diff --git a/assets/rancher-monitoring-crd/rancher-monitoring-crd-104.1.2-rc.1+up57.0.3.tgz b/assets/rancher-monitoring-crd/rancher-monitoring-crd-104.1.2-rc.1+up57.0.3.tgz new file mode 100644 index 0000000000000000000000000000000000000000..b18c30ebc3ec50073b5656b8b7b4df47ca8be0ce GIT binary patch literal 299417 zcmV)4K+3-#iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMR3TvXroC{8y@h=3p-y1N9V8w8OKiDBjd1H;TPK_guXilhP( z3Q`IxB_N_Gpn}pOA}xxPf?yEx|IC2(z2CaOci+A5{omamKXT6A>#V)@T6^W*gMlES zULZ!&7lnl5P#8GUQxb}SN$Yw+FgU3oh%bWVf4#}b$jB%uC=mb3$jJQmuZ*I+0*S1G zlCq+Vg0iBrEQyS)qN0)#2_W+?nENjq9*cuuNM!z(=lbp9{x4||Boc*#;NU1ER+SC_ zpb#7cf%24sLU1?)CJ<;ssjFUbTAML#lX=x z;=3^ti-RB#SODh*0=kxZSil2?`5i=3bbmSpE(i^(0uVGB0f!PfM(6)+Mp+pJDOo8w zNeomF*QRX-LuOuTU`(OS4cQk%L z5A=ZG5jbE+}21cw7*fCn4_5&-l-C<1~3 z0e=Vv4sl0-SOA9t+(7^h!D2xe07v3bKoAO#0dSx%8Uew9SSdO>PXx*xg4ne`I0Q&| z0su4?z4HJ7Ad-L!rV4mK5Ll4dxnnSPiMIqj;aD6dNEP7YqoafSLOh8$Fd!O*C9ELm zyHZPIy+8zl4ghcvPgTHQMoLafmW~dIf`L{b0)*mF7*!x7jE)Y6LV%cEy9hWr|6ifY z|C9Cy`u|-z(ym@00u5rYQaJSg)D--M{>#cK%m1qXax$_?a{tx;e@Ej7EO)&+T}X%| z00TYXND$!L(Joimj(WLbcJ+%7kPHhW0>V82Dcc<0XsMnh6=y}R(LcTg~1UPBP~Yvtrv>GV{sq`2!JCH00agDcGm^q zAaEp(Xz~znBt8%z*a55r9Rb80M4$q8yMHnU1alUE!uTLi5EvGKqVNa;D1e1}fiOG* zgb~JZUMMX1Ge$ZwX#$x)GnhCkNf?#<3HuIzB>^9hV9I`kKqxFBq#{~q)!lCi9iWE? z2*s%aeC8;tT>=Q*C^P{d!IJUm`{5x7zF)cZ2V`J(?cXml#)7{7AVwGEizXO#z9X_y zvI>$ie1McL1PyVABj7kVh?O$MyMw#Ymy{tKxBE!M;|}5=Qa*Th5Q78>#-6ws)g9*S zkcGg4M94n{`aP-tr&Z_w7ws?YzcD#L3;VAmuPFDM_)l3uN#Vct|KHJm&YRpJ zIH;Gjzbu^(90^keOi=E0z90?)gWw>9Xlf@p3JC$EcB%vB5MK}o3zLLmKnM=p1=vmS z2vC+gc^eUq*bRh*(E$Jg;tnFPJCPj(js6{sbO7+1v=jjNyg-Dn6xK`H3x)FGQw7i{ zEKU->lh{e@yy4NF7zhm9eZ!y-2zLn7=O4e8gnk~l_O z3><}&0En+xJQNCoFc7x;?g4=#Ko}hs4MGWYeN&Si8h>UMq5UPp2<>Nf{3%NS;M=Uk zXAt8Lhl1KrC<>3nng78|z7QnDlbEsZd;`T};J6@N6cPsp;(pAD!6UV?<|w2k3WfVQ z#{NELg9S0FfXr?u6on)>P7w1G*^;}f-B}yKhG4&s0l-c~P9X3b$NnV5kOIQO2us3i zk_4pP;Yew$*DrmN&|h9PetH7{5bA{j__WbzL=YV5NvttQNlEdk{Q~6c1A}7#Ni-l0 z;-Jz{4D8oocL)|#QUD}jyW=RN2i%hYMvx-h1H|H_aGt?Z?n(+kD1d`t0Lj2$e?U_A z56}_ZDHMkwa!B z!H=!*gY;hj2#Fa4b_|IKQU!1rJouYIUjj}G2+m9OXHL+4`;uUzKGuk{6oxy0Ootg zFL~x4(ECe(`1eTuIb!`YihqXtdj#*u{kzEjRM&6pKePR1{{P(w{LgCue|7#NFDLuk`mem4{D1TRe@FY} z-1J+BrTb06{#Ky%lYai4Q0p7I-}5kpeu8O`GJqpMtdt>$1Nb}$kN@D0_<&HrT}c5% zLV-|V*S;ta(Eq0y`hQvbOZ~^VL!keA>wofcvdaHZ|C9f({{LIr&-zakH3X0ILSf*+ zMDH!-bC_6-|6v(+PEssU2=MRPhTTNquk)+#z!*G&5D7>E5H#EngTkYUpAlcdKpco9 zP>bCqjd;q7Md2|}a2E=S$Kp`F-#Up09dKeKLI(i;AjTc53OEVxR6T?x0AWuMNBDy# zB$EjuED zz(Fh#XqQkza}vg5!C$^4n$w-d{xK7Oim-m0Q2%ps)`25oaHQw|MscGMpe5)*fc?g^ zKcSHh0RC|+{xwBdyt_9D#SypF&i7WoWU_y2QGdt1{g&C^S^As#{yBF3t7;hEQ4+}f zXZG{YOWN7Y(*CmUAP(|vc@`*`)$UH-$)f(#EBscqKx5$ka0KZ2^Ysx30)Yy!_%@)A zg+dUJox5V+AiPjmoH>XKKw*4-?KHN~{k6lwSno&2kE#WcM1mz#1%xdLz1qkiVLAZ7 zf=3A{*dLGM(P#wd3nFn4#1G^59n=hu#Sy`UcX}a6&!4j|0gG>sQBOeKmV4-zv~1%aexQ{L;@lR z6v_vW2KfFgNZs3;&qyqLlN&-j&Zz(^Dh5c&01iMECDuFJtn89}fB z->-g0`X4+FpXhgt9uPPJ;IsAu0UQM53F5wA0fM1GEO9a~h(AcY5d05l0BsB7@3EE? zU_o?501S>L5Di#ZSzrM;62N)Eu>h3dd3Njg02+gWf*>JQlH&Wph+V_>N6-Ih`Tv3b zTk31;ndwXU!u}O={DuB2$jSXS|5KFxZ~f=r)A;}Bw!8MXi-EhF9OnhW0q}pgNsq^Z zFu*4MMMliAOMudH~hqF#aMuNf(_fz zIuLOCxfbLGz~C5SWjtu-axvsb^=g-9>^HJO7=YRV`DP6u2o%y23x|OK21J1V5F`$; z)X~-jM1Bp{C4MI#`lU|-fFNQ2Xh}q+=;)3ip*wcw`wTD;4ek`Iz8M)13qUa7Ph?61 zyO&uAj~EoO&m98wk@%U7U?2#^;|SmWavKJa{^dH%x7Tm?VE_}9JCQMjOvr+e^%5{3 zg8(oPjwPNV?f$9(`;#27-_PD)KmZ))_4|@Q?euKtDVj1Hu3-90>&_hzC>z$pKEL->(P)FccK) zEaHX3p|Psc(!bsvB#cSpAXp!)G_hU)ll(C*y;B~L{GNsXwnOr#ih(r02Z2sWGzO1^ zBRxflQicFd#3c~7F$9CcBVj)X@r2{N@a|Gjl_~77z8>6#((<8E^-GYz-s|;|oC`f&dQ; z$`^3^z4FL6XX!?e#vehrNzq9HmUtv#C%Jt)zk{IBh@kH>C6Yi?C&EVk-OcdxUieuD zQQrvOK?46}lMwU+fFJ<^e|PV4{r(Qef@lZ^f&+Ip^X}o|zi2D}y>0w0ot5%{BOwTQ zFo?mbJ2|`k4(oQP``7ReiU0}hLI3pIC4#VjVWBw+2dV1CyA}|C&NYi#X^23CF zw|_X|D-P}p5?J(mGT6B@?T$rZ+yPVo62xG=;OKAsM8G{jXb=Rosu_&1FwFo-Bn2;#?Acq9Vu0|EhF#NGFOqj^A}#K@oco1p)6SoHr)`-AxZ_Yd9v zhjaW{{4XmjFQ=%a{M++ing9O&=ik!~ou=7&*k0K337D=M6gl$=uR-}#z$eIKFjYNm z#45k!$@sxzj`>|}o&KrecKz*4w8o{zQIuLi18E;ZKaRI1^9iPsz~JtlA5WeOL}0Mf zb?=_5U4Tlp9dET-ZVJ1%yu7?T)Y0*J+cm6gJA8AB{zEudp`Uq93y5dL(>#@bq}$Ub)0WU)IAT!+?ww^l(RYZ3J)KRMCM*4RR*}Uv%k;=L3-s}s z{oR@k#rnZfF19*1H&bbBmt%6ziAt6hUHvQ|%u-@Mm3EA&RsWuVZOi_-e44UJOePp} z)}x4H+?G|Y%qx?n@?6s~=`$VOnuiX}+r3O_uYlC$DW)yc>QFv?L_2rD<&9n_pGm4n zxks+e4DH;>hXwcnsIfinm#&xhboxF$*|v7{6M^={JYP z-+XSkccIJp%#-@nOZ!UL-)M%O99&n_5oM+=DIPGbRyhJoC_T(<^pv)-^i9>;dtS@a zx+SKLUETvUPlhjYp0l`Lb#tFtf`djX51(V#+ZdBnz7s{IZmH&?#^+#0zEGD5hQ#6& z0ru$wo{t7^J_^=nR4~_`uJ?&nvzS&sR%q=Tt|p+CCQxe?_k7{*)pY_;gs7AT*JVMI&0<|8||4DqeC#&GJ)wS5kdOYgS4t# zki4!kfyAqaRaylP!u#l-khTY{i`H-IaMyTcnAY;fzFJ&+R^sJGIl!x94Io?6j?6k8RdKl&?-M+psALQCBVryC;l(gTf^$DN3N~fZ^Ws%_J*nw>$Ig?XsZ`lV0 zN-VE!w21Q+IabI9mpNQ|QEzj>-MCLyRl3JCH*G<1KkOZSfrn$#z=JqR?X;XKO@zm_ zW$KROA(TtBs9yW?n`~Gpv9Da*zlc8agPRlpg48m*M>O5YJ}ov!Bhmf zFedFB^g`FDEEabtVNj-n>z!p=r1GN^{HvjgSF)R?+%F;p-q`cyf(ytkBd!#NnGAt5 zFPl_EnbNSWqbM1>@K+QirV7Q5>zBoK=BkHmRV_0U__u^4Jq+XC^3?i2=`G28kk1x& zP+?8_GHJ6@<95xROz$RIdCDL`f$V9@<7qIZ+{xIQ9#iQ4E7anBsoL%u{C!=g$6n1f zw})0RCZTBg`pZ&}tKSN?8>%!^&KeG-z@Evvc7wuw_Ofux*O?61+^amy{@DzH9?d=)b5_OaV)&y2 zRm|cHg4C^O`QA_Z+XhMny)G4k2Lnq_>!>AqpIRpCXE-JowN#qRb>_>Jw8*Oa`Cm(q zCbCPncle(kXNfd1XGu-fS)QProK5X|7_zEh)9MQU{8oOit*SnyxPgH5%+2tV`%*4F zj;Gepq28W9OPy!a|3piQ4LJE-I?jCc~AX<627LcdV* z;CThlh3dkqNR2*eY!c$7&TFs91z2ZH0X)MDp82-cad`_lO3Aqe>eSGs?MciuW0n&^ zXw^h`-8dWJl_<=n*ZH0z%4;~V=vWN{-)y)IZH!NL+6Nt$J9iyQhqpZzo8xi@pSL@z zUN>QR6S7dyu;0ij`=h#^h0mw-7^_S=NdRuH;CvD5bL4hA6xO`|OpWBi67Sc3zHP~; zK5vqRQeOBmAR{6#io~z}6IY=7QZp5410__4E}JzSF~?CV+2Zu74vkF587{aEdq_w_>?yKqOxm}H!L z9Vcc5Sp?{fsmLz?)_sQ@X@YO-ydXxIohaYObuzTS-{MG+ZM?+$xtF2h7 zj$rWtysqUoA3ye!Zu@_n1Zw<5o3Wj8b!%^&jO?%LGLrr zqaQORwz6Ngvy@(_PaS@m12alF%7}jKpkXj;rkFB!Gm2K5LlSLZ75_FfFv?2Ae0!zi zq*7d95AHrjPB3@hP5F?Ox@;yFCS%(-!VrBO6VWrzqux^r1{ek;jrX}LHD1PO$c<=m zzh%U)mVI@-d&Id#EhHeyynn#syiL^63I;RLN0zck0k%VP4{Te$BwOhBqvH6bM37f5 za@$kF>(U+W2=*KazqCF_>dji; z3JJE0ed;M}FDX@F>miz~Ny~0PIyyB~r!Vrb1=ovsYx#iUBB!ylqnwJCay0zjFcB2p z5>;fRsimMB3|r)SzP9~4^1z6A_UQ1R=V-D|w|9LF@XBFiLXaPUE5e}eJniKrp% zTO?<+%JuODfdBQj7qRpn)RSbY?lrdQhV=O_Fp*e85V$S4}3eJO+IqnA%? zJaU*`vd^*dPA}xvm6gp|k+z7Up6C;rf2i5F4W@u3>b$l(k_JENafOv2+j66cyy_#JVh5Mp&*@&v z@-oYznSNYg9-&G)_kbxvm09Z%wY9LAcHn2)6Vq)=GwSc%;|~PNzIUH!<0ZNM+3&*? z2ftp9&d@^UK99sYnwNp@p08PXKbBrOQ}Wa6=s~Gymx$CMtk0Ib?#^Jmb1J&wj!@^L1y~RyK(-d z_wjLG?%O7dMtz+Ddh%YoT_9~N(W zEzLOFBa`^~&BiA%qe(tb=I&|F4r^$R9@{^FD?20icYVL=2NxyWYz?zcr||!Ety?^<{p);mKzlWQFFY1D>}yD4g3(Cwlyj zJ%^9F++bkUF^7=w3e%`ClEvhbiF}Ee47YRHB*QMW=)g%zL>1>xGCz5DNqP9d2C6@# z+V+iR3e94;1RE6}eoZe9?Jy_)putlgY^mi3WQ zS;(q$;Q9pyDXv+-TM0VF{#2BE)2m>=M!B6=*5tX=>QAQcE*`I$d{ZyidAn+F5MTpQ zEP6*@ogcEx$-0mo)Y^20XF}4v(=^s~Tux|7EbGJpmrI@Nk~HI9SL7&klv}hO z(fJmfrF!{wGM}PeGcZbZvVM}5tUg@YX5wSG9>e~ZUnfbrzVd?~CqBNrGIy_O%h(?N z5??$b1Xb+X^C-nPZ^-pV6S=rm-vfKg%X_VaNf^$!kwil&foP>k$Rj}JaBvw}^fa@q zz0BcO((_w}j~DJu%wCx*=!?SSgA=5r)Nq}n4Zk=EyO>s zy)D>zEv=|gK4+mV^L=q%AHuYsw66eQr0Xk)Bk3z32nlW>#I)t#bh=}FWFt~&{aP04 zY0TA%tQcV-eZS!?>%;V+GQv6&4o6Y?ZKZ;bI*%qE8%pkSQ(^LIwcH!d$XyBO2OCp# z?!V(ZykrIG=9zBfxCpyau+B0`TT!$GtB`r2(a4*=-hHk}Ze*0f8*!-Ap;+|U*v8#G zs=R63{fL~tjjDmuAtJZ8z4d$0!(m$!VIg~_O`dkIQNrFHPg3%ysjYI5Ho%ddxr@DG z;X*OEeEQmvsq*(hfsWhn*DK;bTlf#|=d9$Oy7$~+>d>jrD3X24@qAzBN%S`rq4a*_ z4{kos$|^w_pq}1Y?J8^DpfEdrgk*EyXVLflhONh1_KF2I`W4Lw7j4*7umvQ%Ueh-@ z+zNl3NF!&~Y<>QNpbOo_c_k{g~zw#D5? z&SG<^8@q-QaR{{po_*o!qL0Ok-v|Zo^T>mCFl*bFI>|pMRFR8bb;xU4WKa`!cHd@7lb0JnxWdual5(^yLn)zT(v7sQ(gux^eW%yK9sZ-FZ#&+9P*PM%0DzMAU`lygyg7c)B5Re+a!~?hqxa z;%?Bn8^a<(NMGy=26BnM+7YGHMIf^|y2+#=h2o8S&WH)BvU0Fu{!>Oz15VAp22&go zzg(huX?po|ReML%t$A1(UBlvPoKP;U(m}`2MFK;l75G!F4FG$VV;)2<&BF?XgfyrdW<8ik#VkE|4LfeYZkp2 z72l-{-?2wt{@je8qC(F*`cKf18qyrrxy#&l=@q$Z&e}9FR=8|x@;z3-hDXw+yJ;KL zFcmRb_fc|Q&Zk@LXO&b44`xX?=j}`f$(lE{{6Lz#kaW~9;}L6YhMW24W5-(GFO~37 z=&+VB>3K`n^4%Egbl0V%k5mlQiL=#JtGX|u;DNlRG^%o)f^SxM7FweXkd8p-C|lG&TG zKHWV%uXg3`=}2~R={+P9ZD(`XJ2-e>>}&L>FM2M9yvo6k_$*+}sLz zaPaHs6)Ramw2R)E=l*B>d%F!N%5M%+G*8bpPy7HJ>eSf@Jr-|WgdTz7p$?z|#&$>}yW{~Tl6|M3pk0aL)EKoUc zu|bAD!~`5vrzzdnr2R^&qV1Becsn0@BYGtKyw`h**t@9vtQ5#?m-4%d+X);Jx3=!@ zdc?(Xv%%G-t3y7hRq)EXnNx#o1HqZtndh7yZLX{l)4^+fY6 z9?80vXK2cSZ%vOjuvH3r(r2%q->8`i9^q2Gs>>IeuFYQP_$u5mQ`5_GR8T4@Z!C4D zP&8zl;Zb?lZN(vI=tAV!8lG>|o4Pl83MHqna((rs-r3eJ?80F_8v3<}xX&7f9w~Vx zeCk1W)Nn;9*6pR?Yg?j{>sMFePd9FsOu-Ba0;w>{C6Bm^B|2(k)+H{H`K7{rgA671 zzHnP3tGO4YWWNV+r)PFbc)o{J4?!*xsbt?ne` zivEQgsWg=f!|lVblE-PJ2sT6e zVVOQ{%{@m_B6(vag^ae+T08i9s5dT zT~NQZ)l?Sz9vSWx&88#B!R>~QV0aM`FZgGQM#N4Dw@9HP{>K(U`oc|L<9v>}nQUzZ zzR<~d%rw0>H`}s)_RE_Nkqgi>tj2*B8{+RG=!GuNmRAoWBwg0Nn$w7?4AQbNKREH+ z&ca~7K!1QNzJ(rl4*@50KG5A^FDpDd^@X=F&@VwsLsO64m3xkbn%#8?=9Zg~$|gD~ zTifZ(l{wh?(NJCrZ{hgF(%(7s_UPrAzT>^6`Yu+lHO&|%-ED*pHjCG6HW$8q!rLgz zc8=-`1zArQ8q^s&wUARf8=X$+9~VZwf1r^rHavgx=rxt z#0{qmyv5LvrfH$Jy~$PctZ_(_M*66ioy`Sz;~F`3`6~w7R?+w1DUQZbVuaL`kGmmcKok$ucY)7h>wO(=;|h<`YexH)lSb=%1wZj|UFJlNeE z@bn35X4ch&oU1<0&e`*hDr*~x@;T6SG@^JNK4R0uj6FRR|~S(-n&n<-94>4 z_Br{z`z%;XHVfL2uil)ao&{$}XTe(9p3ZWvf-JVxn^7*d&LoN+1?>#!2cO5p+xU}s zcdc4dDtZuH`T_Y}mu>;Ql2ab<0;LLes3p}gXTn6Ku*T#%rfW%8u{O&}c9>acyGiVZz?Q?P=GKUFffQp)keOC=B;TEJE1z9$)qKn0u z!}F#&mj~`CU$-S6!h03aMT(!eW_1HsA7}do-u)ioH|PK5tw(j?y-(8>$6Rp?V~Q7; zJhb>86nLl11}Pa_nvyVr+xsY7nZI0dZT|d)R^7lzL1(7XCO z#KmN>#ioPL&Qu_VxZfyBBH!Id<2s`|9-(Y?U2kxW9K3#O&BtkRKcYphv%wZOZ>-#s z_Z;eAdm9!N<1%AWRoa{55k@ODr>bB-5Pn3ae^usludB)hRgaCW8J~azXW>0y@N@ai zf^L4#jH0#$wfvd=PJyLk3we`0I;j`TGOW+%%`rSM5fS?AtxYG|^R`XJ@phn+K~<^E z)v|XfULm@lc&){F%^n88g8Iv~Uj;KNwNKsSw7geZqH&HI>$-Mn(VI0k|1*FMJG+)p zr_jy6$Mn1kk7rzX;B@d_o7mQ*w+R>aDCbR=(@-ABxzUcfb&lJEdn0+u?N0MG=FRhs z`=1^CGQ_O>f4-zN9=@9nB#!Q#v7 z=RoiAQA6?m{SMI=kIs9>3$5lQA6;bk@9$6$G)YW&saILAR97{m=H2XhqnlA?ft}$9 z#j7ympf0AbkC!akPjNclJXJrdAE3}#qiR4OKCf}@0h&)mk^TJM<2Eq8ghsC;t&e=~ zEy<-mjgpFu|D-B;BRz&)csBo7{@1O*)%ftuufr?D%4^*L$@%^5-5=V!gQh)uH=6@L zKJjd_&uZv`-b(AAbef-RJZcsBHF)cFEsH%ER~7&bQb!cF(gw4CBwLk=8H?QmJfZ4C z08czSeo=Ay!T{c1RNVO%*{riY`Kr_c^{n$O>8$fD z+McHCTm=EVMAXhCifsi~IMOc+$Hd!sPe+Dn1tb_f?#L{TCGvfV_&XyW^|pJ6wNo#xKCq^!BTC z{bx!o*Q>bsdL`hCCT{F{Jd>#>=VabRnV1Wl$H(=!4EMjvkMey&#{>Q2pTUTAH2QgM2lA#?V|a$tM$XYbHW zaRn;X!Ba;KFqTGLcsXSjtiK(Qd&6BQH}$3|_VUy)zUYOYH;&Pl0m&5Vb-wT5nnCW@ z;-{re7qntI-(2NJEX_>P)T85v6RcOuC$(w{y&hn7zboR5>IemIRad4K_>5zh2 z#_Q7I@Y(yX?=ikeB=0fTQgfj0F}H}$(p!OdIBeLYOLEp`z2QYwyfxy{)xR{jAkFuTLKCwKgoXcks!Uq^cg3wQIea(&!ZD zX~>e>#lJpG=^QEon+R`9I*EB zGL30sT0`H_wvyK$G`Tom73;u{lO8&6ZCS2J`*t=%EwbxFfk?>FhYsDU-MW10>eNqI z)~+qdJ7a`0{EH5|+CQ)gh%!w~xoslgoDy6wcgCbk(5#%{10vB*FX6?NJl6-biA=^el zFkdJ+h+)+8;guI{$UQas<~k<&=CQYg-bJU;U*_*v@UNN$DOP4pXGeObm2>Lbc~}#9 z-d>;$lo4Y%n0=R7QHD1QVzr;0Vb4pBb2)q(z?OLb1YXHE(yCr+MD|(FlY=!k4A(pJ z;`#b1Lt|CZ#?7$@+_y{}4}`?|846+Bk@$1*rROud9k|}u-OS_P+c)b%HbLh|dMhfJ z@>t;|)aB1}ukf zR3U;A(CC{LZv(|SFX?+rky!96GuQ}mbO`cMR(O0!>9jr-d;GXZps}TO$|uKL2P#oR z>Vi>E=_HKHR~eJXrRD11uTdNrAK@62HK^xtyEVn#h`DF*MuRcg?Xp|Ap5AjN@Alir z-tvaoi@QHQFVF;8olA&Ghx?Sga&dJX*n(u=dFNdA_SSHRie-!UT7&^M%BoeF{&DO= zS>&^U`Q(!M%wEz!whzT*H~5_RZr@etx2uSGpnAnE^paV$7|G03;O11@y8stX)Zn^j zMW@{bSw@`I;I&x~`E)VeaWm-FisL-r$%XdiV5+ zLgY+sMhNCvY01mi=?6fO4vV8*&*`iSDCp%b#R9PfsZtL)wyX>C^Slbb^rvYhSsSio z_}RD)_Z;eZ__XWVY1UYqPd;03jXB1)ULMr-n%Wm-v{-B_@T_7CSqhNece1i{%2^Yd?9#&iw+;}59bj8-L` z)1!${U4A~8Xu@>IS8a~#(+W@E4PD0D9D%a$r~m`W}u9N1Bf(ATqU*dO0O`&#FA?3l`;1SR9cbu{}6^ zj%3{?rkiKQ-2#ndIOTfzFoZQrn6>hnuOMuS;^@(UGNyYhg4Rp?7FQ%XpN|R}i`wPL zS?Yw2oNOu_SI(3a7=RT;oUax$l{6FT`ut_%Q-a!N;ROt&qCNd?5VOjZt?Pg-MfuIa zDjao9%He`0nL|zvGIFOpLoWD&4GmcVkOtW?=f}+zBM9L2hw5g|Ad6ZJjsov3x`LEt zp(9sy+AV7DWQmnJZYOX^SYO+_lkLTquZ>!(ua>VpseJrN<^)5O)U+Ua_LlHM4YJBU zj+VDXCW7D(sYrFqNz+5`zSk7(OYPNhaXYgWP{c5yQ0y@(ed&wys*n?O+cy}MU0>a5 z)m^)N!58hASUfSp!iVW4n4C}Q%o)9~@xSV5$TG-ESb^-jl?*Ipr zfb#{z;9K6uSeeDZKMLPuN8;gPkuBY zr-%@`OrNIr#N+5@t4eJ2A;y7L!!^1^xg(aCk0;+sHby9IZrXZW__W5mw@A#Z9Td{3 zboNcbZ>@WOkh79M7C6*Pe!C4GG(5W(tin7ho)iqgT0Z{@Os+@2UF$)p)z*JCo%pzy z3cXQ9azs1l{q-Z-6Cd9d)4lw9HsM~>h$eT$aClJ{&&#i)*6M-vnzR+6H!E=?u&Z&y zyqsbxvTeB60=R>gSBesE6d4@4Vot)reUF5ygHDUexAlIiwi|lYl9H-}-~GB9I%YR) zxYuhosH=R~i`U4~4}&-DJcP4$r~fdWMCaW4iirN!*s zDeSSCk%EEpiNU?5#bPSUpSox_uk0QqX*}&wHPCeY*m0|pTI536xjTA+tzwC{+7fVS z8r__<&P>j!8zT{N9}8&$loBy6GilDg(-AC_vgcj;-IW1yf!%xpwZZg$Pf-Q3F&zCVddyKu2$E2T{X}*``9O7q=<*;+t zyPk5H9hy76eVuGP(XSPgzj`F!`|DKYnWA`uN$*f&yE*@s+o~ITMti%m8oesU7m7xy zGgJGG1;gAubMzM68gAS<=zRJM6`nVG&6~xd^bUK0h5Eko01^$h_hk{sK$^QHELybt z%Hws`xJY8>HuBwNi1j$5MJW&6pl;#1R0MqyI&$CAKHt{+jaC%wMj>TU^hIYA?zK|(?FNLm zv|gAE>b7Xn#m=b@zb<8DA3M{thh~$0JV2qs$A>GBNv*toluX$_h9%~Kl-p)U* zxatZuVw_j2YLCW1iBj)IAoCF|-IO9Mg$C$=C}w zlVuYTs$`PvsaXC{b;lwE=&&3Doo>< zYP?voLC`O&aE3iG$>je1&BZiC5WD#X$aR-LfD=!s)yOUGxNdZ)8m?7s&4l=tFsBS-t$ zD{}Rg2*mOdlfkgmJF`=iGgluDLR{R;Ya2tc>(hhXCDN{U8XMQYG}(o3eA!$O(u1;- zX53*;?|x5~-W{>RNlG#!PgVHpDCgaCCqA~`o(a7AqP6UG&HjrXex&^p4%J?kTBOes zwmKe3-1+>q_+*)=8p8oAU#0tPi;#jmpD`!JXd|=Oaoe_J*cmvp$R2ltr*fFbm*bIR z!970SdM3AR(KBOCLv#zZ_39#2c2zDecN&qi)CJrT7?Q`@;e8RKj-dHE#HVSgh(hDTyB}$UsC_2op zuK3>XoZ=j8+}mF0yY|kz$syDo+fO@O%5dUlZnGtsH!d> z@5;Sq_0nCZSwn@*^?1$akgme6Tazt!Lchkdhkst#+ufoORLu=v96c|1k&^$g2 zZE6axR0|oKOg(e(9JJkYIe96UcDiT0S7HSIrja(S@j2?jt^9%LJg=d7$>Vj|3rqOd zoHvkinCXPinI5k^xGgRXMb|pqRW3hLMO}4G^QBET`<8Wqowbj_8XCE&UH7qPaG&ZE z@W{&~p_d0}Z|Z&YibcR9^}OmJnTEN8%aed#pq{(29N#*ggvtJ8J+tn>^?Q_tDD zK-qGQ2_cN|;aB*g&(Lk#UBTZ)Yi?$QicZYIF0vk14?UTjHlC#1P>4b;^CC+>&o|(< z_sfn-ZLb=RF0526--s`v;)#KsnfWqYcPc8F>ta4*cXi@?9^m+}D(-%CoQI>4+!5w7 z=w-+ueefP-z>BrDbMWdy!8Z-|EslO=ZEeGdzXY5Wm)?Gn`3lGGgm) zj+K0;c7(FW_SxUBON(>;?YcApJN++p>Bj~d9OJehlNZnM8M}Dc?>T1dHE-WV8Faso zLse=(O~kXzncFT(xu}*_wEU%A|2pIAdeJ8FlYz{i3*OvlacK!^KHbph>GOPV534_Q zvc`Ueb4k^fi2k?J)LSb}mJ5Z0hRJFLLmhXmpLw-^LVP@aY~X2QSYXJDHe29^I!gWO zlh57=R-xQcZ~PFsU(r{HA+h(u`EyPbWWFb-|n>qlFOorV%@S;%1O2xM&%!81^Dy6%15R( z+lVu_VGk_o?V(WE7a7N=sHJ<3E#e(l==j72)p)>Z7B3MpMtvoc1E*Hi85#9|slzhr z7ZlR`T%_?5xk=_FVo9#B?=7{L$P}rU$W7WFiz2RqLK-4+XA<_U$xE6qw_e7?<3F)V zt$(tlWG8Hs*wf!GjqW)%EnQ|Wk*Tl)2TrY$?c!t9|NC{= zIKIDKq#$zw|1)ZmYevp2OLxU@rVU_?0wwD{l$)r-ei@P17q%3RGp{aM46=wV147ZWnIDu_H z#mV=slYw36T8Z;^V}i zIM$bj3iXdYecD(Z*#AQ5)>+Q`iuZy>+ENL_*4f&U|ebyLL3l z!^j%Ez}cYLi+G!PKCI*SALxoH0s#>FK2MAV;kAZ00<|@gFM(q}R1Ut0_8Gc9|raOu+ZPH+{2pFZe2sk)b zi6R?tt|FbQtRf$9j^9W@AHtmXy2Zyz){-nPhdy*OOYGL(F9{qHFXI_@^Pi0rLjJRn zViC~HxVx`_R|A(Q<|?a5<|=hZ@$dRA0-D(i2t>XkD{We$L~lfo9Qqq6!GQDLLKcQS zGKaOOeDD5w%umD~zY}eV7)8_FCBYBzAL_a+ethJ`pi%1WuIt=+sS_(x5}KN)fvL@| z7_P&~E_c^G$dz#=o4qd=mTy+A@Sy?Bg5#MGHQz1W3X#FTNUZKCnHOfNbb zpg75;Of%7FFKvT%USO8Bza_PQ*Yymw5p-~A}D4dI;DVe|9~OEe$o!Pk&lhYL35OPj5B-Nudg3ZT%$%9oB1c z`Fhqy?XfC>ml=a=NslsPc`hG%Q>4`qPvdz(H$m}@=}j@gx9?%*oZUB$XxF6Vw{V2Y zmBe29N^y|7>yV@XXXX3g?)AQ;`Ab2E3c3_@GNh}$b%C4#QFVhfM8NXW+zY|;WotZ_ zb540GogS&~nID`@N!486%8?DW^LjLNH|~0U`K^;fqsA_>k5h(Mcj@#efr;#*>}RFtM8*MZ3~pwmpv zvUOXyeYUkTyJ%90?)4+y*M8$;G~O8h{sW}lM!aF8N6ET*q~2;gXue>cU(j@Th+EAh zE*O1)mP=e$-RN=!ERw;Q^kEMV3FkUjT48ohurv7JiA2hT5%-&>lW zFZ;h-(meF9caZk5ERMt`y0wn_&1b43k1=(6ZF@+pa_x0`4Xgp1vs2- z{|->rL&LqUU|yU_Rl(xuKNhJO^?!64!ucPahH&N=$p2qDWq;HJ{cn4_Wd#NDO#j0- zKR@>WZjsK>|DSdGA9nEnQ>VY)MfokZE<~48`FgnBX~ah-2M4be)LDs}qu=Dvs2Qeh$~Nq;G3EVv5I`2iW92+awNS7)LF{T#!kEwv%fgxoZIk$ zR}_;s(>Cm`Fagf-ZHdh#OFAYx(_VenWn$xAb|Rk{6c)P+Als{y94-}J8X8=`Cxy}x zYZU|T6sN6x!LiM5+d+A%`PjP|X6#k*$O^FF%5m;I)Og>MJ5_#&#K`&U@6NOLwhv-Rj-=D4&OG_p%UOA60d zD2lG}8-bUWiH(lBVRWmAa?Oii3l2SJNm$KGbhc#0SW5>NmKvO~-ALh#BfBb&G4+;4F*t}h3 zy`OcB&RHIvc^;j0-#1x&>;3*XZR(wMt=m27o*sww`e*uGyrpn2(~-MK#BzO)H7r|} ziNT=m@VtKZ@L=4q6MT7qLP%1u3>0{AP=5p8f&3(*|TOfe+^W^_~{l^@}JAU-mIk%;xV|ULE9#K|yej^d`M||KVmh<6YFc2Nj<#|hzexyq>g?IkB?pXRI1Qkh_Qd*BGXA&M6gdxz zo7A9F&Fw+sG&SmM;*#7-=-c|(5tWV5eDUHf`g96uUIXt@eos*#wX&mlad{RM?}*d5 z%JR`5WDXk1*!jr`F4{!)F-}Qz_~E`_CqIZpybZwV+omL@eBu&aLit*-LquMz$Z3y*^wF|#IS*|ziB`Y- zg0s_A?j(P=xcWVL2Q#p=GefwB(??x4+%1hlV=gR)1D8*)YnIYv!fOi)h&dX;4pYI7 zRR?SKnOEg*pWqEQ|0V@gc614ED=V(wJU`8;f#%y zI&d5vh=7Op!lDhKKi$R#^Y|WGznA8mS@L|(U;|Vm79mpZB`&ILXIR%|VO^+4zov=U zz3yo&e?ZPMvM;W(^;!H#rR15WlkbgTC7*{+|{4uRK({cE=Hqh=;51J#Kz} z{_poGrkS5U)*k`C`;hOShq~VP_tys|zn9Oby58@9uTgb`@F5OX} z@A+tG4|@be=yFL;`#dD~E`YliWALXRB3K_SYbpG_n3z@-*g`!%M~u|LCio3-ZRu<~(EY6=q=Sp-zk=#UNa>rq~z9pnw_ z;1WD(R^x6r^Js-J7hqrd#)buN#bfL(=l5ggrqjNunqzj2qJ8>%w&R!ut=p zZyHuEO=K@cL(NF1tAMdfvj4(q|0C-aO)VV~&Hv~0;>W(4Ay?oVl|Uyk7!!6z!t3se z==q>Uz61K53h?P%W~afYU-d)y$TEgK3QLtqL2WHAk)38++{6hm4R>W&j*B@az&2KO z3nUcaxDldNh-mR27X9$Xp8vg4cvSdSm8h)iWq6Xd=~9}$hII;RY{aV@n^Mf(!~Jvc zP_R!nO?$;{M;tkleH|7yz%fDCe5W- zlIal?_w1s~t}P-6sm5<8f8n2j0c5fQb84^W3Sj9n!fkZ_q^b(IBCa0Q!2dx>uF&_t z7we5|&3AA#4_roae!W#%ONenyRA4YBNYt$0^eh|ceh|vyI7H9FKcbzHdq+TSe1KWB zElbGStfR4WBtC_4O|zBZJ=Ua261wF4uJIp~EQifFf;8R!^2`5d<{jlrS~(Ecp4lQd zPAbB$dTGlM;h08cTueCld+m9JyvxoGy@*?D?cDsl1Y?fdg&vM&HS0=_3tQNuP1j9e z=H`sW(MAn>KzT@po5`9>7Jcf%Y5xSex1WUM^#vNl7~v;PB;Rtgm9Rda8U0|X%g4si zFp^jTE$lqdytsi6OBZ~cj-?x6K7`s5SaRH)7LRB&Yjt0!R~?NN6WFN{uDN64q4^@b;&15%d_6)*R+r*RMIAqx`BLw;Q9eA){4{nx8%Vp z+F8{=GDMcG=FIX$OS|P#B5Vr>cU4+4mC%YvASIKD%6Sw*Xq`Uz`!JA|0UniZ+(F!_ z)+6kM12J(_Jf;R!DZq*H*lHG9LCJ7Q-9|<=3_E#LJXo}Fa^>s>CTFIEThv8n)GG?* z_j-k~n3P=ja&&&L;QYGdC}b+#K}aq@lk4xQRLRh$XKQa70NaMTAZOg<^9qH6o{3J< zCGHqsHZ{`R;Vk{6Iy1ww;eyYo9pT36KsR3hz|)BGK4QslIjlSbjZb^@k3jN5T?pq< zc6efj4BFRh)r!QAd(2IRhZ@rvh}4Ma4#3zBjQBcdiD>%)i`phCbddX6Jw1z82rUly!p^@v(WFG@lb%qNHMg+2|uSS41T(!a@ z!$62Vc=wFPwx03cY@zLJ_lnus@Xx*b;DbnThym=1rY4k~vW|*fr|z^CTnnss z!SoZQG7C9lDH@4ZrKOyZuOK8lUbK)zOjMk?D3Qlh$)`T`+X{uqVO+99hNSmA<45d@ z7gLDIHB?pURt*HU8-}#QW@fXX%>L3CrqT1?%|85%|9biVs!(TTIdUazS{F5p+L+3w zPHE|x*3oE}ytTk2tE_XVHgl-ANE$g57(&Mm+x=Vq^MmH;e(MS?bjk*Jg0Oz^vVV8S z)-b(8*_p9qt*djffL5D`NUYKbZqhQz(PtY>0mu`?jKZcJ1Q%1UT-wzO3IfEZ8_3x* z^Rw@8g-V!=tz!z)X_qawF2NjE%3s_WoG{$MNbMCzqm}4{_cCpQq4U8vvh2E(3lP*BhAh2^B}kC-)@ixAal-JW5ma;=`PS4Q_F>c1$|p)m@HVJIiy zKq~eEX(EU#0gz~%N8z9=DbG5I(J&MnPes}snz@5KDYNFrC|cQ}uDnF$Wi&*U#FMWr zu#P_cxSSyM;>R8&K+>3mG}%Oq2U0fX{>k|fEQ#=a&MRhK@%d`@A|J{OwBkH#UlUJ) zE4r(LV^cVFz9DxgM$!!!V{YNVK2JATBMtdNs=t5+vGJubH3iKw6sqhdGKW~e{S$q zI|JRVSkzjzqw+3(V&%Q=q|d1$yBPJH4n+~x5YBN?%IzunI&LEtTG}kZD}HQdXmKNA z-`*@41EaZE;oC*Fv^F6Sh5g zf`A{oY%!{3Qp8kECCKa>wIPSTDZk)hNFPK!Rtk5ewm~-%0kU<*7V9R;&$-nfZe8SE zh6jwUZ&8Z>di5yw+v{lQghDkJI%M8cjV`I`(ee9Hg_y*V$9_-Ql&VzJ6f3&^j1-R0 zs%@;nH#p~)g)k;CwDA4)3$k;q$0z}P;j!S!dz3Vx_0uA2w_)73BL77010@OrNGt-B zHu+*!S!>jEP7_1&JJG>&$*^^V;LH(YXwxJgMSAzrG4CQ>>71(#L!a%;-+O8PM=e&k zJ^2&ddM{LP{o>lq1n2wLEib{3rdc;^em%ijY`j?ymcXNFT&LS*+ens+gKLXU+?s%g z5p(UWPpV@c`urzE+FhnYK`ri*;zyecDg}5aTi+jnn+V!T$`qz<`PB$D| z*Omj9`Je^=s8uN#UkAk$RT$8R2Ina0;V=T0#_ld(D{M*DKKHzj4_4q+m)0TMsw>5v z^Y#9NgIC8k=60NwfAXvJsdFj+=|(}^wjI%_O0VvtS!*opK49G;bhxcpusL2vowTsR zwh6Tru9c-)vuCo^Vj8EbMygHwa?rS0LW9t&5oie4;(6Uln89kUW9_J z0n15o-o6Xbn#FB>E$%OM)XCa?C_I=HQVuj7fz3Rq?F}Gs38k}5FyfPwwYRByRVutriB|pDd&l|{nF}z#bkO&QE)CJZ6C$}NA8nLEXVm=FI-#XHs5Q~| z#E|%IG;HM_%(C0Xb5L`yWa#a4F=f?F3-@HIw(gA{Ac0B6rKjgz8{JbjRvXj1;uRtv zU<}q7l+M633+o&m`Bj32 z3_#O1O-$Xmq)jDZVj~Q!=o!QI`iB{^E-IHb_QzjxFAXa*Xv^E4drIb_6Eu3Z!hjKV zac3tT&_uwLh3XZMODmMcV0*Rc*eA5wMAH-VVJDPL2`GYBsuq1!zPHi1sHb&Ie{ZbF zL^!-r(nbPb2@x7@p+(K7{B##N{jx-{{u`K2RiTD@e5X2yIAXBBp`h-y+*9Mby|Fpx zTVs1;k*RJkxIg|$X3r2!(KU2ty z$`)JeeID|!8;0DbFEtHuv`Js9SopFs9JSZ}*Fh107$d-$*aafRQ8?EJC&jT4oNo)- zlXT6RX&{FW!tEJr85Ze0T~M~3ikCSZ9UpsL&ogR$g{U}CS;x%l5s@?pj&MI}Pa5#9 zr?gPCINuU-B!=s1Of7)~wTERjU5jlgvFm9r*MaShr` zj|fAtuYGSVA#V2Ttw9Xk+1wKMh~~FT<6SxcG1c z4!^l-OwVls+5R@47)DzLvi((_Ag~BLvJ6~mvzmu3^DoIqM{*h5Fe--u93X&V5CK7` z`JqO}@?UW zcJPuU>hD=NqAH}C1=&<<%P!C52A5FZaVqpWM3}7xrlN15n}b+>c*&^9XM^fAdJhCMcn3|9@c+n=m7wu-^Sf zA_+qLZ{i))>rhZWn^I8zf3=}@*Y0(Q@QG?3JcW<#A^s|@CUYV4K@Ho-3a}X~*g7Y1 zyc{xoDG+paTf27|`-@QX%$yPJ`;^J5IgOj4wux)?-e92>Ea#TDUW=Xu{B83Mq`A!2 z1R@9s-A1+r1zLYa2D55Wh;YTc*sXG;4kDl<1LCF^4zImAs^}7)r)_3|ce$CYwZ670 zaKoAu5e-3>EM@35Yngt3j9ng5F^@-wg&rlPJ|GGK19Ua(3k){sWISVHB2ud|vkpCk z$yr6eirDaS^9%VpcFeOdROPYb6Wk^LoZIS4=L?CWy6*43_w3fxJh_#irSnp8w5dpM zHXtg|qO}rtv-P_0i$TsH#f)Z6OFBiZ>mGFqT?xTCH;#6ydh3+(Wl^ps0LpiUd8D`O z=Jnk8uLud;e4a_4ctu5Y@(Ih#+q0a9JTDd=LPee9HIN+j>`rPQSHck-73?|*!W6~Z z$%81zZru>v*lRnGp~n`^w7T6FGVjnhV&oC!JJx90T_Z9CxeWfMV3y~F>LXl#Y{!Go z0UIAGn};4f2^?8!xo)FJI@eZp&H45S+{)jwP5C6cbSwgH8VI9jEwO+o@Eze`uSO~ZPLTlLN$`!*AJ7PZf>9F)ARe|lcYeVI7&knE@e&GMsta=C8ZS^aag!i z&H#hLgru;N61nJExZ7Lc8N?2`nyL(U0vBV#Kd+R0ORWfCqiD0kjfEU*ASCKhki?2z zY6I+Spwch+hDRl@?B^@PbWvsY-i*yt_J40jP1y`0+K0N8#_%QFs@Ccw;z6`5e+J%dX zlk(5@y-H;7n3Gn(=B`{$8!4wzG5RPgT0A6St`knG9Zm|bM_yyc*TOMnv)ZYoa${RZ zrZ{$Hh}zzUR6Q}77K{=M?L}bpf;{B=MFf_h0AIxtDVRcJB|UE1aHk-9AsaU4&&5O@ z>}l}Nyp)6gEy=*`96*=9cE>e)l7RAI@HoE8=^IvLcMM4nbN$3K! z;IN!xj@%2qZI(F~ivV-5If&c~==}_9{XO0}1l8S^4Tl*!NyAiP;fTK4x5)qvNOA30 z(xBZki>kV<{N7rhLZ$=Wr2ME6C=J6@G0fm%M#FxJXFA3PsqGQcWo=-;OeL%-6)b9? z9P43t?>@cyRwvqd4x-G}qyr2Zgf|A5Hr!vB(Z-nY7nU#Nx!ITg$)Rr+sZ7e=h&Cwf1NvV!a6I7p-Mnnbr>aQ1AEl;*L~P3zmR_ zxxKzW+XsaFI|WsP7$1J9EQ9W4mmvW0m2l4XNeyMvq>eRvGx8{B7RhPzVXo5f^XSr> z-@9=~dfj!hgWOKozLT!tB^*J}&|o=v*W1jJ+hc9>puDj14Z~KBbf{Xg#8#FBKI3TQ za$sw$?scjRgNS8_!tSQO3-qV{3M(dZfmNg~poU^>ww_Bz3xx=iJNu54ig%$MX>X`e zZ}tj%S6iX(EI<0`{>oWth>^8PF1Ir)U)R~c5`Z&XH;xs(S`0mKBI?5cS8gBj48KE3 zagtjW-8m@H!21UxKlVQ-G5NkvE|&U;;=b?V_5`%~KT5yfYVVylUp;qUdF(&wGyAaT zdOe>vd^uCc9h|yaY`T=}gA9P+C^lO&crdN2>d(aMqKwf0AOTJgF0(xh$^K8L?}d){ zM%Wx@bFQ8!xI-Z9n4KWD6c#KZ*md%AdpAZeG+vB>wB1OH5AQqrvwN3l%9*UxB#;!v z`N1mi`N7Gj91E%+t2{GPQ%hV$T;}>UYekOhqxE8#S?$+oQTJw}cAHb2Ka?dBlF@wtQ7lUTDuct^FH+Y}Tx&)BOfGTgy*80joZy@1B5FW=w4?UQ^3vxR-iX`%E?`#f>{$3aas>2UALD zJ7%rK5z7R+6e8KjGVlM`*u`%Cy~j?{MdCZZsZ$zRzqFc3k||2tXL~7+CJ>8Vo-L4% zv2STAj&!)=;E7na+9=i$!2~ZQqmv-zSUaF4at1AIY|$}nrzH1N7bQAm7d*g+&DDur zoy~FyTix#XYhk30W<#XZJ2vIJ`zxNH56(QM>+f$bVMBghaviMf^Tm}nZ0pZLaYA1E z=If8z;qMd{$l1A#uj<`KQbFv@SlT;+cUG1H$RH22f@=@7fym*rC-K-eCA1-Ho2^SM z%NLz?0p5R_WSotaWHNpcigR)N!`mv*bYz^R-Kc_~+X=#Q1}un4qtrF_W?k?40ynA> zk4tWB2Ug76Cr)Fap z+BWs(2~s(`7;tkKf~@$r8QC4bxdpi9&Yu`qSVxETT)vkJZQr=}}geQc=0F=(*Ta*16bMc12`cHgA7J;rV~v z@xe@;+_u&?+r}2c#{?vhnHO!Ssgu!{O3 zk9(LCsZ|0Hu631M1A7`L=}~r8b#j0fR~k-R|A*8mHDKa+&Do03fLI|9p{&fB?ms|9f61b((4O6alS zGO?K#1AjQQ9GmZ9xkjL8xH?tbtU=vIGQ4_>f+~Ico-Hh{q?;_<T9~@6paSVmM<{I4pQN-`ivt)RYIga}=Mf7tXd- zx_AsJbaTkgFzc@Eb&9j7THT}eTzrO`RgHZv+`O#Gxng{4%w%i8YY<2DCTq5e%#~c& zd}$#V%*10?Z(JCtQv$XrHdrPtg`BeyYqD-S=HQH4;Sb44T!gs2q)L?&B|3?75I>b? zCzUc_C}sOD9p7O`s(H} ztYf5XYmAu4bbG^RQKAI>yqc$r#yogV4oJ41Iu0rM1 zo9g6+D%Dgfq%>xDQcYGZQ_h0t!qHAAw!=e<4|T8woR@KsN*!{m?Ok+SCJ^ZS#IsEh z%7XWz7rG!1vM(=m8Dg}+IHq(e2CEYTON--@D65x0U*s<4tX_FaIS2_9of>>O@RCX@ z=0YKOMWxkTvo4F1#fjE*T!c?OgfvGKW78}KVSB|i3bZrf>1Y3`eEcRMa5JeJx4Pa} zE}nG|M`29CwBTH~s^#F5Rx9A`;_zI#zEM1$7yjeGn9l*^tlfIKoVsti&sgn?BeL)6DfzHLE-VV zNsCQbcoOcZrFby&nD2?v6C;MFdx*fW{e@>&!}{i6f_L>N zTrM2fUQE(kNhKOZRc9i;(r*WGE^ooQT|NEp>%*P5Eo&yObFu3KG?MLnf2t=ZSOKnc zvCT0G;dY^)-t#jwq3!&KNS{;rqo700~egePb~3e*Q}6Zgk>T>ung6gw6aTO(Mk}%urQ%hy#a0 zxD~-#_lH84!E>5JAJ4}TWk8kbfe+4`-NBRzDpwj#-L5x5oatWK9fNf>U$znQ>hDrr zNnT+7areqz!U;u&IJU{O1gZASn|el^o(fT8rICrHQ@Q_nNL=@~AzK8CgT)I?*hfOL z;BpsLE;2ziPJ(2AgL$RNMZ|+%=oK(qqWUVP0RR(r?SCn_MABK|U(s|ivWJZH{n4fF z|FJYMAYdo_{Cep-a}uN85h6;J9S5I^HbN-uaEUESr!UzT89dyGcEp4J@3ZjRd}8q- zinu%GtZOl8l_a*Hj(ZsE&tk9USpubVV%9Qbj=g_?sl=3&Ng$?swp0b%Vm$e8^~?!F zy6P8`d!PPuJ7IF=On^~aI@nVto_~&;*^))jqYiYfyL%3<|dBxiaH%(Q%v0dJ*yOsKc&E*D6* zavAcvwFVzBNe)*?8MkB{RiMY3%t!HFNKop`4}`Q8DUN<|2Mq(s*lo{5(UM@*0kwa~ z2_J2Z9m%1>SvVLh!S&y%L8Ts}HX zFv-G{Hi~F`$ZqsCxadGkD@Au6^|Af|w;QeV^6vl-od9+rzJOwp_Y&WZRk!aL(h5Sb5UOgNd{nRq1UiB>y6{CD;6NMqFcT!h+zM^OVXpq z?Nj%-uo4D)oH?N0Ih!59)a9=_{GgrjtRRs&Dls00VkT#1)Jpc}%4e*5nyaw05T|X4 zCxK9%M+f2#g6b5EsuWf>pda#6NwH+-uHYqeHyXVLPpv)?l<<{Tq$z6(vL8i1O1oHA za-;AgQ*18!$1LvIxWj&ivso^xKMKCBOkFO<0fo1f@AU`#t92JW0mJIq?t2z%tM4B9 zn2e{i3D2_c)WvNlSHiI|{=8smozLsNqtj5DaNt)0UwrlX6|CVM{<1-B1_&K#QIVvh z4)}iex%|`ba#c?Ub_l71rHrAcct(d~1fdV?BLp{*-+DKbfA}Av{8sNZgWgScG{+p~FPYqJ+ ze8vypa>hd=EP?kqjja;8e;&vJx(iM?h<(2Fq~#wi@~J@p6?NXaT7I@vXA(fZnA)LfZ}P^sTwHX*`ksaBKwG#QpC zR#(S6xL#Tz( z7~ny}0Gedze<_EcgNdsXk`K3hyIPny5<^-u-`&_jae_#5Tv}`JGx!`UKkVDu`!koZ zN5%GF`t<0&BYzjR-jhxDULRMTp1gkc^heD@F2IVCfjKNM_#vLPMy`RBYLG>o_YG~& zIbb3cp`!h7k%b6*W3`=5dwiFj^*&T9nl{QsruHlURnmnqSqWtsmAw4yGs168N*}PN z!~~8G2Q~rFNPDYsmGQ_Vw8Ly@5x$qK0|{BWZG?61ROdqS1;i)A;yZPi3Og;6Y|UkP zdH$Si^wKCcC#sQ%(2MLO!|RF zLfW)r+$-{9%V1iid+leJ8HF5K_ZRY1jy`d zvD?#U9OsdXJf|_c_N@>KO+<~3+_XUkyGsnfYu)?TCpNcWV6B3tNef|S1<;X`G+f8Y zK`IiOky+44rzt@=f$rn{bHmA)m6z13mtQ|zLKr5jYkQLuPIRk$BAU5p@8^vQhgcJpbBC? zdx+!BsdfBeFDYeF}JSW>lV_7H1CNDUWQh0CR2RsyL0&O|!f} z`{q894~@M;G(EMb!b*txkRC#lZ4Dv7ayn@(RR!4!hC*=FZ2{> zpY3~YZLQ0ed5&1sf~==Ae|Q?pNug0wehRO5uQE}Tv@-(GF257}A_v2x1^JlRf zT-$|>>ccOI$a$9DZqkffLF`&(?aQ2)!0cr{^r`2%fVdTL-$`sVWu)0zDQTmen5T;u zY|JR#G`ha>3#56%_(ZUcc?aAzwHlt89ov8x0p>)qvPf70n3#3*?>YAag{=fAc)ugK zcS0*Uf~Fa-$kyOs^(X!v7iSR;EvO?h;8bW1nSQPY7z``z!ZsSvF(zwcEm5`k?GqnN zmI+1@1-|q^fN{1K#qUJ4Jvi_iZaeh!dl~r>_Mzj`Di(WEN_r?WluW|+B*)I%!sGqbR$M%0mC}X(sy*!q` zvS;DHq3U`pe`QhJrMihK;Jtj%?NJf2xhU@Pya@2$D!&ixB+6L+%DwtH+P84$M;%;U zE+!)&9Kuz1JwHjumDne7NB-5qIY_4qJmc($mmffX$zWLFH$KJQ^zGp3@{-o!4>0;O zz?G;5#5kHH?4V@&J&f(*tZ`39`P(@Co+|T>t)y4x!Pg>luESlq^n;+uM@wm(a|@dC zjeG?^$#)MwFL$B2!*2~EkBo9e=IPDsRmS=tSet1sI$KqrCQ^%!q zC)>rcrJ7MRY~RzT>xqk&uBd|s#al`;C=GRiM^mR^Nrt!fh(%k_A~t}tyyo<79(mq3 zPL{bZA7E9sW^~kn3n&hH(C5>4Y`sHrBikb=Px zt6rM9MQoVV4+MBk$bd70y?W5^)=dMi7EdPQB-~v=%Nf6^<3OH&)=F+OM)A1(6Jz&( zPIFK2dt4fkS*2qqUIaq@`tTFz#$M8W_2uq(FV_wte9#BsQQcd5Z60QoQ^R4iZ>ZM}Z zXF}R1KG`)A#gh{kn6$xYEn9I3-H{_Vt@Dj*rJcQGT|S8T>p86}fBE(S-eTZud1~|; zBJJvyzrB4_n1cD!cKEtpgsQVf`!>AVxCs~4eJa1(>*#$Y^i zJ8|@)#EyhaI z%c^X|?KNqKVZb>U3vyCo1J_&LY9dRQ$CaCymCPTPmzQK+y5JL`?m;mE2PJcFd}TZX zC(>4qgK3*{Q`S+}+GwjIaJDPBxe|4<;%TOtb`Eug05z#ocdPR~%l(`DJW9HwG7$gM zmCW8VVpY8@SYeV`3d8BzTaUOn9k#WQSWAO4!W*qG;D%XI`1vatpB|_OaltSa8E6)= zLkYl4l=#cq5(u=N46j|Sp)Bb}H(z}p!YnWuU`PBbo;pE22%)P&Gm5w#h#V9pmKuH8 zYG@+HkwxFKzdhCls@_ZKNQV&x2huU$E~6M`pKkLs;6h&AzXM8+@b&r~e~r}t*g?TV zo&B>!CpDqXV8%Dc?>20JQeqj`>7z27aLfQJbBOx)d)x`c&*$mp ziQ_LT5XvYQH3_6|2peRbS86$RC@TrRVI=L2+EI+QmOV&%LklTK`eveu=X0;!J0{iC zSX726VWXy~%S!>~B#Jg2W`LcTL;RvQp(F>iJ-Xj4lB;bIam8i?zR_?dG9mNQNS4z@ z8g7H#mH>5Sdc3^U`Cp06_-Kx^b8~dNBi|L$HIm0`o=!z-McG{5*Zsp!`1nAa62xfG zWWO(A{zyfZx{khcJn6KsV?*U2ACH~p1Ovp9tF>@LSnLvS%LpMJu1YIe8jXAUPQ-KX zGa#%6Qr%r5$oXINXNC{CKNQ8Oz>$(PKRdDd-caEz`srESX>wNE_z9+l3OC$vu$=d`XrOqI z4m=#cwilY+g@Was&5kG4q_8)HZ)1mn(6J#XrfiVOpeg`gK%l>~D^3lONd`2i1EY#_ zYH#uBe4Tpd4gj+OHm2=pT*=t59Wfl#K zYD(J`N1>$gOL4@i3mh)SwLg+EeuYc_-_YN-y_Ag6w>#{Q&O2&a?^^7Cp7>V(``c;# zZxxaMX}EM8>etL5Lu~uwm)md&k>iE;b+~j~^Ioop^Kc1r%l`*4bZ*496{7Md(Up{1 z_qrL0>F!pd&SbBYmB$(oR`M9wBiL~pWEZ-xqfR{SMoD=gcV$*Em?aX9|vvGqm_}m5_fm;85hmHC5to0dW%j% zv5jrO+{_FfmUY ziM02wf6#KnKoVx}k@rdWTAa@wW2GTHv`%k0$bDk(D98|laF`88VLR`7P{?6wE?rd+ z0ztbs;W`gOv~Qv+6U92I4fIi*_D1Y8kZd?>N~0oYCMbv#A~0a_*8&6lSAJDpZ;Ic8 z;od9^0Fg_7CCtgBBQipg$();6tDQ)tu9+En$y4dHyz$Td_CM=2E*?61*{AG1!}S9& zg81>*1GjbED5QtR3Mx(264?88%Ug6Z8F&A}3A?(pEox8FHQ_b_I`BACT6ImbO#bP_ z;o6N!X0$8Y4PUHZRqM%#>7h$}^c)-&_(B9l(!)K{I?tJF9F(r_Q)=v6RdQGl!~YW~ z6YzPqbbB7TxOkZ}^MAi@S|U2VBl73|IJ`>w7&A9FFDu*oIhj!>s@3}$pX2vs+MGfB z1}A}gfm zA9qrAKu;Oj$9_C=HOe!@I@*_M<$U+@t~ff+<9~9slR6ExEaXH}qw0`5DsRTt8qQ$_ zdnm++<@l~F%85m6v1OyqsqLR;bp{rWJBXVh{eq?4-Hs!cY7}h9^rgq4v!(Ly`NJ_5 zn4l)?&ZT@8nbMrHG%UpCdZ$7<22S5W{l>tgtrap5$%tZ4DAk1533;#rS6)OuV|Nr7 zwh(tVOCAeBZ+#uDAqiI5V#PBoXsnOvi_yFl5$# z1~OIl)+$Z{6!I9|bj(sUA+%;QGITQP+E86Rx*J^_pU}r6nG?m4Vt&fm!`GenZ&|T> z`l2#$^qS$^ZOwa#issG9vRvdfD}E;QFl{X5qV^9b94siFVLZ)ozlTxqXCSA%)9`1M zRGn#8V`xjxit;ZXXtVuE&&v0>=tb4ggUG;w97)#S0`AE`Uum+=K1W3QSSmAb_HCZO zTCmTGw~tnGjHstRyZ)qnZGQCuWFsBV&u3IlNLN^4%$nqaBdbEJ{jkETDv%QKR$JFY ziex5Amx|REc(rXzCG%HD<>-BLPjw$mPp`dX5+*SX7A!=^dKCSTLE+rafr9Z0GDnFZ z>X0iQXGcZNIq*n5G5+>%3^nQV|37x^skavgYV>^FU3b^EZQHhO+ctJ>+qP}n{nfVZ zPV$^HnVihkx&J5YW+f}%H?TgyCDaa2A;WlDtrE(vOWS8QYEz9}{dt`gf&2||AA+S! z?Y*^FepA$~Ms=!!)bRulx}xaBCnUovYEN8I6F9(Vfjr{?u-6-2pyTNLl_m^klg=<5DmKev~Cy4_yR@2TA>nH@@5dK(no zzU*Yk87H`)&J`yHZ6MLT6vsJ{olNmb>EMP5G7#W3l*uwO6dv9huXm#D>5~tZfN?DD ztA2z)N+zyJb?#75-D;HT@5bK-RblWw!9%sKAFmCr=*MBF;hbZcu{7e7;3?IjDNDx< zbkm8+ov&gYvDZJ5p6BC`@DfhXHy;nbvhPd{Sq}~n6JHXbP0Zvom)==jdmK#&2#GK3 z(tHfBzSJL>TU=Ns#L{f!>Y4=AKUoC<+YCl}=$XNjNaNU0Rau zb*T{W0}V>8zS_TSkkh&BWU1+v)njam*ynQto>z{ko8+UxprI4yXV92VuS&*c-qjQp zepa*NbC?(QkH)b(Nt>0J#K~Ihc63#zFZ@qVLzgdYM&dL+*Z5{0fnorG#_j1MTEX%y zb{YdxWRbqL@Gu-9w$EzN&J0cLt0lsj9|yk!1z_YID;`D>cu?yq`*=2} z>qe>ha{E@_CA#&K(*$)?>KZwufiphB>d^|s)+C~32;og&O{`Nwhizw7qht64g|YZxTWS|E!pcGLOtMh8i41?^c9X(F` zzvh8`3LzN{V0m`s6vt+DNn(g*JtTlfJmN`!mpJg=eY5~-muWR z#aWOl8gj(mS#yNJVrothUeZc4#n)@25k}krgsH?=BkyNDVyW# zP0WPO>Nn#+%=;y#a%zH9Lt%W@M`eQ8O12B+x;i%`)=5nIA3lg30J%sO!Jil_lS(?I z>tB;emVPKVdgRy8+_@Qr7XIO`%n3esO_Qx?XMq<=7%jn_`zNEsZSZa14wzZtIm|Cw z$O1V9qf&!T2MA$8X|#5^zptOb!QIo&l96<^^IK-K)|xNn)|N@Z8^geoi6~?NP!1hy zDs^*`LsZck&EyXduARv1(=SDmop$V9lFt2abS1#!(9$M>Itk>v?H%&P@oaDw-M+7c zV7qRSXT=6;NHzhL4=?C_Zs_U#n5+4>yx6A2%k=oBim6YlZ7d`~e#Ng}%B?g-OuAgu-GuhZpw-=~Qt!5v?kkCbFOEShV{ zrb(}gsy~r+T%4T@V`9z?N(G%l$UkrMR{X_v?YigGO;=M-5!M9Jhq0H)#$3XSv_AN< zujYM-)WlM_2V6R~D87JnLkqd8ORn79?5mPBUuWM1f81tV?9;(LccCdbbt>>KhU-|` zRi&NyB@4^!j05WDWxJOQOhGP2HuMxo*1%HgW`vnQIW#|%dPEIjq8ZxOTp@p~db<$K zi{NdRPwAA{8d>fLQ$$S zK1UThXJ9gK>XW1eK}R_ivolISkEJfz9jkkaxDSZ5$OaY^HHKs&*+b0|9cE9ki1j&D zmd%^^(iEkNyFr;yq$Wqt_MEx16Qz%ZMD`3)vu{`k&Y2Xp9LDNg;Cn%e)TwuC_p(HZ zY*q)1GK4u2Rt`BnI|JV)`V8*@9YE07#mUEe76+<0|`D<;7~^X4x-icE3UVC|U*+dPxcs9irKHO-U|3 z7>6KJ;Twx4Ms_^f)ji*mpk!N6R(zJCW4BV46)_5Pj2f2SUEmERoT{s>I`$(C(^Cyo za?YYLDsE$W+^Q>_s;eu0_v%LEX5$oGi!my0;cE?3^X~95D(*2VZtv^=b-x;`>K3c& zX2Y!p;8X)(`k!c+l8ZgDXYqV!n40(f?@WtvN{+qPF6wi1Wrt zDOBUmAN^hm?b->>$i%|4hKiLjrdrySjMwrttXudGv@4w5^8kV~6u?Qm| zr7`YBc^m>oE{$`}A0%xiZXQav$H-ja%1harwtvx06%5!Vl8TOK4Rgi_jA?j2yXO38 zZo`xCn#lwaYg!2wAAz-?ZNsS8#k-eq`gh>%dQ6 zhS>aCsJidL4m-jQ?u}_R5rWdFI3tRQ^@?C!tln9u)BnJm{fWLIZ+-8AxF6t@BWu@H zhZp%uyMr2fv|-ojRO@im#MoAokXMpi+1K&u>$X{StRY}vJv)i0r$_8iZY1#!MRzz3 za21`!|1k-n%Cbj=HIZsM1(*Zxo#QN(DX{QUj9{JhsZwuo5XlU|E3ns4l;0Ff{Lw#@ zZUiAnL9PG&2IT!D(X^hp;g(p0h|#jve37WhR8M08R|S48^eiu|15c#WZS(+f*cZ(GO`4(v%+*`i15)+;5st`eg6R0Gjt>N%$;xe!aYofUtLLx6svm-dU&Ddw4^uW zPt4%01W&an>hQ*RfNOm6NA3)GlKmt=%ZnH#fypX)xmr;NQ|$-0ngK;j8n%fU!0W@o zxkAy!8p(qK?u?TUMv4a{ftl%ET4!RZA0!-QsRkL+%!A{{29xVl-gh~=%)fL(|D$lI zoKV8|r5FY;!!zL!W~=_eZX++zCmRzMuswH7Td(xGbyCA%Z2*;&gia30ImfXsw(62n zO4bi zG%H)OFEQv%H43uo+aSX*>L8dqoy+@1sz&u{8q(VJT8%tCA^3Lc@F9P?hO^ruP^EXD z;M>FH^0(obj{kQWL9qX(kC^U zhB3;I5Stlj`fWv^Ipp}mv0Pd9VR?t^AUs+agZfKV+uFZp{EUVHQmx5SgSrBe{;5jU zOjdgnP6Dwfv)ieGG@BEfmD|RtJYI&*#08;8ZF?zUB?ZeO{kgG1+-m7EL)Q@)u4eo` zRoDZh{$y7DnTSft#Qeft+E+$v{qLyP0C0f$=hQ)G{lHv~|^y zWyv?5^AwP6Z%>u*jVKooFB96TMO8vE>+zs-?oqIg?jV>!A1osFuoxrj4Msw`A9P0& z3$52<053Gyzk)@MW4=ay^D9Ft*ohM9w z#CQ-DL|iViY&aPPLASLObcd#faaU%NH9@B<(hjXP3c*4$Z3Jc)%-|xJ0*KkrQ)WwK zcQ6ljBO$xc4yBVhd~p%YC;Iq2sxgbAHp|Q;SP~s#`f^ok<){x2=#qAus;TtSN0Fyc zuwQO=5;9$Tu3?!OTonneCi77u6SHsQrw+aGo+~18Hbujg6~$2b^3v|W@2rf!c~ObW z%%A*J!wHt-6(D89Lfmp)W@7mlVLx*)>3;aHyVPy?>b>ayB@9%%Z(5wzsx8*z4eRwE zW!G7vnjcob1-1h*6!H%-da+Wk0~(f_&qnsrF~daaw_72G1?ArYSuwSM;)B;Cvkb99 zXKP>YoZ2E;n^dQy&D7uBC-WvTzeZd}V@=6MqG!XRM?BYPip6~)b?$qzsN2|M)F4Tn zIs|M`O2aTs&QO=)frdL{8KzekscARikQ)U^O7hM>by^s$7vU#qfowPB&L=2+VAXq$ zuWl+-Q_aLyGxz(Kl`$rkc4MRoOMN|FFL0B|nj?y|MpA7}VLE1^YTn()QRfILZ#XSI z-$!_JwxB{A-N`dPr*{Ih>^vCvy;ykWtrIIiy_^8v8=`7UjUwevw_|iK1`aUvgcnjm zo|aHTuJr1t%xAKRi%8;+qOv(Afb$HVL<3Jssa6yPTMZ3j?#QfWDK zJq959Sv=9lLm`<~^KCySr&|5VU?P= zUMda0?YN@eUaDhsYrPW|w-})mETWA2#*rHlDsr%f+1iC@Q&V@wr zROdHhc+=SmTZNY5=>Oz3ae6*{?v;_>@2tkK=r(d_Vd##qT3d+bhNzqzwY;;ndu|QJ zAi?FQMA-pmW7fs98XKV>bOV^>=uke#Qy}US;=RWAGA6zoGP!qrn-ZA=3=58 zg0BL*{XX}<{q=jgd_D;$;KU@!+eP5nF?J=3bckl)#3EuHBXH~(dg(_yM9|nVZm<8O zjh{jk6~A^2v)CcJ*1Uyg4#J61{#-gjk=ZqI470BqF&wdO%Mzpn4tt`AELb~!;@Wm5 zo~?%Q?w!<#SJMu-!?k&PQb;U_ScsP=AUyN8NgFf3S`6Idrh(VYB z%rJ%8lZP!iK6e_5xv8(Ln(t%pq$e04a9{ChNGIn5x3h!?izqV`M4ztS{fQG#W{Zcp;5wBeoFH@N;L`EjwZFoL>hU|nV$9ndP zeU*Feb5?9D@(=UH({U1g)+?i_PWKv>;@5TOm)s+Kc0&FqPJvI=*Wc;kw6FW;_Sf6# z;ZUB_yW1PToo6G-TZEx6IG&XGiZGn4+u*o7sxZE0{;o2nDXXqzF-`p^uZ4!buJH_F zuF7Pwqyo0*5b>2Gao1JOo5iRO0MQiSW636f0tB$vadm720r`nGS(G>aI(qRG8<1wo zp`_I{zhwju5NSCa)4iqd;Ne_rf$w#6bUoigjAPiM)UnLyiR8z&XW|M?xbO90h;vGQ z8Ju-;0~Ywg1rJp$m^>{rl%jTu^v_0j&2pZ;Uczoon0eXyPFk^UWvuL%u>=75jQJ9z z)%)W~zH)WQ^XootJ&h3#|`Vd+hW;2P$w$e!D{1b zNr6NbwzEQuviQoZku?@2?mCe+wHlh~@z^Xi9zO3*++n-dTag9j&SlkMga? ziw)&<6RW?Ic)-3NWmog?J=X6E=hdVPF10rLF5uKsU{q+i}j59Bf0=n2^w z-|2Br7dF?)nSGP^`h{{W(OQB2oNNe3qOj$aL=7bGtcs@~$&YHqd#?*LY(-LdYRuO9 z{Oq)HpMim9wa0n+cD+dqnT`0NVB>cbHUgrkl!>rMs-D!H0S+xB!iS99+m>NLq(L7*t3XS3X5i7f0G= zI`e#)%e5&L=o2A2HMYV3a+3Sb<1tFlPTFZp$yB5;5-D*BQhd7cxO`7~OIZ@D#kFFz zCtTO66@@|FXl-=Rl{O2$Glk}({!QYa4O?wy*ji1ZFv8&X5>rHgzep$EQ&&JRn#gC3ol3`(6_0 z98O{R-%*YbU4*5Ri+!2G{y>y2B%R2c>4eC7ji?V~c56Kr0}BJsk3SxAC8R;7>z396{nY?}AjVtNgoiJLK^11U#Z z%C>UztV@S_O-DQO5K-_ot9sLC+1<#n!rHfIYB|OxN1pVEPT9QSdsrj_Z;9OuSpOD9 zjFtBCyQ^#t{-)Nx+lU|pB*xj-?gEqI7jjrp$h8RBa;GjIat zG)E$Hj7D2iQ&Ju9luB19)s93sMlHzo54`CF%Sk~eTm`8{5o2hov=Q~M%JA9m^umfP zv^MKQeS^Sml&qY*9-fYD=K7ON2g!Si_whk4aF~2oR?tR$lyZ^rWMy&Kh7vN>+A-@L zX%ryT*_)4E#KQZ8PV!4dM>9mfpN^I$I1M{Fh~1U4Ez)N>-vjpVe~h&OLhr@ho~+W^ z>fL*t8zA_^jUz^&VJXJ{Cwp9#>^GWG5nSNXR>95L7mCDG>zyb9K!3W$5)(CDF=yDp z&DB_Q3!==H$I0QpMy9U%#>y>3OkXkJn}7iZYek=4!L&$ZWGAUz8jjBh!7G zKjwsx^!;#UCt4!N zdK#~pUci)a9mCvDn5Y>jm-9csrn-SN3FT%2Od6FB=EUkd7KTxTo_ z*{fIiJRAk1vbZ;r4O$8_={-i23B$pZl~`aIHiU@WVy*Ct_pd)Jlp1`;=Ua|qQsz>V z)eZ1kS`h!zC@t^EU|<#bJNnWWeR-B*Ei~xn=!4{ZT}mQjmMp|A4%0i$nenQzUujL9 znRXxg^`nbrBfOC1?D?M6YAdxms8s~SQ<%#p7N`>hnh-1O08^8zeWgN`E-T+LH@o(+ ze0E@RZuXbYKF*)CQs8&@y9ADI2EYhCV{wxzQ)#2Zsan%ejqy3v#NVaRxKF7I5qfIV z@GUHo5*Pc`pj{Hs#7$C00nULYY@ZbZIXtp=DfgTX9u8lU%%u}jw%`XFuVa2#Z_AW zpZIShbpe`T6vGKaZ?f8-Bgv2MbB5)ZKMd2q5Q<;tU2TlaZP
#HL`<40o z-PUniFn9gK)2bAhdvBjdZq7OhMNZ_6@P9i!FZ?qV~)6uHRnoWcwkyX1V7DlC` z-dr^R!TiC>kdkJtw$Bc$46EnZV~74IWS5&M8}@6jI5DyWr<%zsW>wE)t%gCR zQ+<;4K;s+uMZiGSPHln;mK*Fl1o5_M6@0#O(i1If=JBXPH77%tPTWUkx<&($v7h9F z<8N}KIlCZD{z=eul9g6G|+Tk~dgTHV19Sl&;AwDqG zyf4+fBh$Pk`vjH*L6UNd$``G1Lk5$7V2Iidu6FhG>#N}7e>x=%=Uv%-txDR>cjNhw z{{2V){-b~Y(ZB!b-+%P)Kl=9{{rivp{YU@)qksR=zyIjpfAsG^`u89G`;Y$pNB{n# zfB(_H|LEU;^zZ+c{!uAF5Rk6?oPK0bcai%(8;j9Vty<93tZU3Slt>m6q$EEFp|J~F zmmK*Go`Iv27En)yUI-o1*ZJTnxPT2@v)O>HUoN(#oeE$Q6&5RXht;pD-5HV0bGezT zIz5wq93eTAs;_!N%GK({@o>!Al2c5yh{r{^5qSPa^8n&JGc){X**~ZFh<~df1OyVU z)hAZihd5yanb>r?&6{x`Kj4GNM{*&( zDRX`^kp^xmX(<`3JVZ+1saiAUhF~Yvj90r!mp?B&J+f?)M{~^tJ%;w4S=Q{$3>90J#pZ?mJ+qIYe zwv=YlZeC=bvwx3Z$+7|Q#5sP!u+7<&{ zYE2^g$yKX00cv|MIL7wDuOhNDG3o0kZ1&Nv13_t_8|A*({vd)0Hj@1wqke#!nD2^WlwW+BJ z2iC>s0lMkB1;ugmytp-tqT~pA+?o5DK!5X!A135@KLz<7yDix?!geUl&r^ZjVdyoZ z#X6Kt)%dhzG$KrW zg-?RqL`EQ3i=lZ(!l*LM^S1O4OR%glnz31VNaTe>b6nTVng5JO__@0L4O=Bp9C$SA zNI)IR)E<}QXI~ySH*G2m`VH2EuLMZ3o#d5O#<%Dqw@=OmQI={qzsAg~yxc(UdJJ7m zR-)BbS{)<30A%2)KyA`nQrpp9wMHF>PT_Hw!5-RzetY0}JQ1^Wl!uz>eLGqfoXy!7 z#(T`beNlg$g>(Vdr0mt7z_TPGopGtN^%j)0T{?fnri=&47C@S+37Xk_YW7t&Kf_zK z?8;BTYbqCfxaly>x9{L=JzAdEF{1-Q5+npYCfHh*VX1PX7N=kvHd^CHgN!4ueTxQ; zm}dy@PZOKOzTsC%T1mxXb!2FgWANnKEN%UeGnV4XQ zA>VKq{wC$59+h#ozFErr!ND=!c7US%LtVMN?lDNWc`9kBN1#I4inK6l;8a%0>kITP984J?_vd4K~4bBEXCg<38a9TfJmayP&MarRVIlaH;}-NxHq zuEQGecjFraVl!^mbc{Vqt*B-n>DHotN7OlOZH7Lb>_?-YvD}hs)kR769dDBg+-IKWe$i zrAK#s-@K6kSW_=HIga?EN8*D{DK|>*Pa)@g6I_kC*cHx=d@=iH@!cUd7Yb7t&*lDs zD~kpzy#y;YM@#9(nH2r_Z}!g}pbP*&FDu*S&-y+eo0IcTAM*d&zlv&-zroMT$;rv} z`%Pol_igw3Cg%Tq>ak#Ff2)%(+ihCoH$&p&{ycak#_d$Uv(P{JhPaJ^CorC!4f6xL zq!ZPN(-CUjQiE)pTURo;xShb%@(wC)@t^)0EO~9^gLG1x$<2?~Q?o8{S$%P6KR%Uo~e^(hA_M`NPNEJic z&+0V7CYwiObY}W%@%UJ~wmLY$hZddE3S+!BS$FOr>SvSA*13UrHbt#G$M4^2Z*_JU zpU)NtGkMcxu!cfZ2h;!|c2#0yv|y`~fc!Xu3~RjcR}ib!ih3h`#g2g_>dRzqO3YU& zk1cQ2oF7hscl)mLU>5k~$9+h&3b=BW%?gMMIs48$Z)<=@+dcfr2KlQE`nPtrmt&K^ zjL}!7HB*<>W`dur8Ma7rV5vB4>=SvX9#uWw3g^MDrh=kszdJ;F_}v*jwacc;`QJ47 z+oZL%%h$^{`N)5#dE*YXWkZ8#>e4~kbA4GFe<9D)>aGXs@=9G|*~Wf39>pUt@b zx>cU{^7}LBH8c-T2!!X_(0fJ_J*4mB6*2269*08Je5{CHqnaJgHA{?ML!qHCyM}(z zXc-IjZLpS>wTWp%niUv5Y2Tt3U9kmM0P}6(B^;?vD9Y+ltsZ##i|Ekk4!oTi zd~wj%(2KcZM;?;}ovvUz^kO*Js=ce%OEfo58SrEnMMGJKSSu*b+JJv&Rv-mpSalKU+&^%z8s3)!LOdXNAg`K&2*bq8@h+9eLlqL|I?qM&(#bK zbZ4cmAN$Vj+{k>?A!>?xPx-pNgY=lBuYQeF8n{4>_e_VisC*IH5zM zF0Lx6$vk7KRK`F-Hs#UiDE0D{!%{OuQn#M%7Igd-2`$jK%Uiz@`^OcpxQ5+6k?L{N zK_uVn==?Rz_xQ3M{x%N>e&-fhQxj$V?^}oF3hHIc9_$He%RzBtzJ5BxF*6j^AbVj5 zmv^Hc{RmR<>nUEd?Ucxb8%Jw4*E{Y&j!K-2;{o!R#GJ+f+x-;uZ{#lYxvqBPIFLGLB+Kp90l!*|&sI1@yk-!wYuhQE8?9*8DvdM`JD0a{%pr2xmDNcs-!(mVL)p>?Nh zpq9MpD-lzytq)POgRp+aOX-lnVz|IhG3`C@A{O#2Evllv9rX+?TNGaa!ymcA71tRe z*6L5x>RlcECw;J9lY~oHKl0}0pYimoX*6p}`|)#ujzgf%iZ?0V7H=0bFWf1=7 zZTZMcF15GZAr~!zSt2d%?cPm%mU?Q9O2~7X&nX6+uOlrZN*$doJC-(2CZ^$SO~Isr zU832OD`> zju-Iw=pBq;1rRWQx9Nu*Tf=k%19k!J7FmwOb*4uz=WSu818l#lsXn96`nGIvZPoa{ zQPm7}?z{S47-~C~Nba^U9%#<|rrPaWvnaA*^4HF7ya{Nm_XV*cJ(N(J5epO>1cmm` zLBysje%5(oYel%1wUjvvMx;>;(8*e5OL@84zFvl$OJE)M6%CYiX*G2xZkuMn_DGxD z^7n*U>zk!qEu%YuguTepN{2YMzCW)^FjyeOn`I~=7+b9aKisDK8|8W${@J`=09FMx zAvR5n$!Khf=rkwGXN;9vZvZL>CvYV>d2f8TAp|IkW298C(c~%7nH#&R*!{hptn^N7fYD{Ssj&J4?LYbV16y1p4LU;UYSN`7eMm8rl%)X{+N2>1>Dp;4!PLb|jn=uJ zR@k&5vk4O0IwP?mwue=0ChhImlU`emk#Zx&<(0P;;PR#gm9THUzScP-v32V#i{h$4 zdvmsaS8A2rV@tcN7X^>47vpR!H5j#tRm3hGJ#P%GZ%@H}RhckYJHn!Nw% zoyX8)<|;B<^^x$zUf3yG9vNKl7KPYbFKCaMHuE%MkT*7dlqY2McRCt-ISP0M89&&Y zP3?FRIh`a)5fK?IXRYnb8M)xBr@S1yUGqdMcGDsoJDWZ*VyEYtrb>8oTB+XRa(7fb&bR^%QcuaoE=cKz{$a)s zRZ)^X<|-8&<~8u7s)EpC70O-HkSHW&!I zUWO+~A#5MD=v%#9WIl7}S9%W?yPpA_WV2=JYHhHrac|%J&w}1G@#ivr;STdo!(aCo zb8L#hK)2Pm_oZ3=d!s1CP`)=11pW>jat_}Ma+dX?bE8;r86^SpgR335PyO+7(Yc6JBq{nlLAwkX!FBQ4y{&_6F{EtmDE8H? zHpRaM>g9J%~|Dt-|c!UkhI(PN;!sXL|w5Hz8HF@~h@0ZpXXwb9FXP4zn5W=Cbp z1qHB?)X!GzZu;#XH>N(f+ADu_krAzoQNlA|w3(5|QFZFM4CWemFHWzv>@QhGztG9( zA5CZO{=J)UI~D+9HW&nc5i~4V$?Z*>0+UtyL8{mB*gfem{#F}EEto}}fZsUt811#p zlu3Y_4S~d?oFqlV946CXc@Bllb+_c*ms9r#rmA3XVju^1b}EuyrT4kKC+M~|-0Iz+ z+s9;WGNx@IfILuQyg%c>Rj!^Ad!PzVo1|W2&QxHx`lZ@j6rdS%y6#t#wUatZ)>z|( zX@Sb0kzySIxz)eXYZR%#{SgS$$m%%PGPiispLSDgjSp!wPT?fQEeu8ZhuT1%Qh0wt znq|AlSXxy8t$)tNZ7^^W<8CDwqnW*+zeAjQVVNAb#HiT>-5HeQtU%K^;Fcg=x|rL`8G>eCQvmDnWzE&pQR;N>0)U1l#0pSVArF;)<(G7l`#MKLZk13Nn$b1gvinwrju+7#t zJ6B#HLmfL36S5nWIi1G`yCj+w(&SQ_@2?fm;$*9&TS6|=8r17~B3{u27bev*C~3la z9e(Oh6GZwp9#8c@(YA6#)CI*LS-4@cbgU4~JY(Ci@2=Hu2+bHrL=CMjc+RE%>AL#1 zjPrt!{r#>_b4YoSnBzsGv%l);x6(G?nQk6bl@rXXJ@3JD#e^T!gw4gIo!n~qiV6t0 zjujuQbJxb-@OF)u*R)O)A*txf6c^rlg z0GabfyRUs46F0#d;@OrQBW(eR{JYGYI)ZiLR@wDQ!{G&o%Pa1Y5^sc7m%PI&4U$Mk zDP#68ge2BDcVCzaUgtIdwNhElKW&7@v6?s}kI)S)s zuhVuO$v#%Uu9yLu(hd0T$Ye+NbMdhM$>nz%NzG9(OdMZAa2)^}i9fFnoqrt0ILI!h zTPXC|9RW^~B&2>-JbOc1zrgd`8OE?=hn@eovGPcF)^gsxacINXkF3K*RbkdWQRq%}^V7#-n^lb)k)2=IsCt+t^ zaU!vbh!q|xNf1ncMm${Q*p}r9;*hU8l7{C9|E|o{z1(Ynjow~*gBS~;`(*aqmtJaq zq^(`PwhObf#8qf(9uNQQcsWuJ{hFPvr}E78W|!}Iq7X(mc|7a$^zqf0pdkbH53lu8 zyyv;R4exo?@6&u$8+76%SNKUuad#XI_9a)G*&AeuO|J&XVRpFum6vL(uf?K4$E@gYj0`gakA{fFzTDToP#nm=jKl*$w%Ur{pkWF*B zTFgp}@BKt2w3@l-JW155BCAR2m2FS_FnfSZ;=Q+R<=^Z*7YO&wzTy>frB$o8XInNy zPe=rBc*UFwERr7hI1a=DJrP3tjL@3C9}Q(gmv?%~x>3bvL+8yxXDJ}u^7j>< zq}Hjf2I02jdc=#eH5H7Buhc0va|jxU))fju1FukUV(VTlZYS^>%K=s);Y=oVK}bpw z0}wZ?1)vno=lx%zUDe9 zOo6z*ymY~1ns}`q{2|+r4295@xrj_E`kG1W21-d@aKI4;rL|8eE|F|zO{v!i;g6Uh5A#pI%9`7TX7&jx3*r)*=D zv0)^WkO>kbll-~hGF*J~o%ASD2bu_*TRzF+!v=oRZ3c4<^!M6=Yt%8(aQCWlqO{El z@}l&bNMpoFVB{OZO;mv1n6p^`ufqQUE+gE}*)XqfpN0=2TXvF=fW$V$mEfv}n0-?&wX{c3x z_VsN%N&227tunai$s;Vyr$6ck%n!Q;uVpiAc!V-&&)5B}SNJF@` zuXf9Jo2>ZE|}-?lq&LDjrP6m@zKYI(W0s zl<~XM@+On%*rj3hyT2gS>%FuQYH0Y*gidLD{M~QfY9gC71%b9{c(a_M8+@BRM&%V^ z>mlW?P@1e`H~@8*3Ab9fW4m1cV;b7f?uORTUJ;WAvt@sZ!I3Apl3Ex7vf=iz)r@x| zHt>6`fwskfnX@_40C#u_@wx?8cilr^k#yV3-k9wLaHaonIq1pO2at!yNx|`vzovB? zSyizZq&iF3#ZwY>dz0_;F$qXi`n*MShcD0foE_0yuU!C5K(fCFPkjFVh7FO2>C6(N|0XaV`E56ePB2Ggopb zXKpRjNXv=)Lg_I)bmHt0lYG|y(shG21jQc20%xizRin_4{rhOf=0l#CRqJ&BCew1x z`etU&*&A6YnL^ism@;S*Z%8aRnqfn6O{NgPgHXC~vQ^p-4yx5Omf#weB^;r(W(km1 z3;HGY5;sQhp+aQHw|4jyQepe&g#m6O;-8gPhN{_|Q{k`|tL=DA;2FAQqc!mSLOG6i z;*}8NrPZjd#;fYXS|r>Pj67UW^X=;Dp|+9s@0*BI z;O%p54ZkqTDiqkj3e9vqW3VffRe5d%iEyyN_MnN8QPHSGxjU?Vfm)>th$Ji+50Uot z$aIp|>J6-PHxOlu{ya>Tz5+`Q+Hu?z9PDYDW@-qHEl0ext-a1xu8y7Vu!Vj74QCy$ zYMu6kHBtr#HFwp!YyMDOE%%U8!UVZ5`_@=^x|i17LnmGbvInL{)(uQ`!)~2T>uab5 zu0}BFSu=NVNzgZlv}&up`fmlHq$XuGfX3ErBJ4b@Nn6aKX|J;&9s_v|)vhI3axH}q zWQ@IdokgX~%aAT?4BEDwySRi7-?7=bJZvRUt--9-%5x$YLYs(A-dyXk5?1v3CU5LTDY*`H5I9;(%(j{#R_?^#HX7PTkI9!|zYnNiGZb zt0JMsORTy!Vx@@Xf+|NN33OhXjj}Y{B(e-f?Oy z=0&9NKBIQgWptaeXFE^;I#9KSQaD-`>-NR&e%L|Fk{vtPHWX`DWDO?!FGoAv|K!b^3TZ8>`Ct-?*N(+t|&~hmh0T z9~6!YPI0BSN)ztw987-ZZSjU2J6juZYs!h6QQp6iBB})=L>NKII)E#8fA|}4fp}36 zgC36A=5*%+B~;IdwW!aJF1l1<38u(EC~I;H2nqqGQhx zTpqd=Pm|2Ybd=q{wlUH=cp1$;@vWnxexf9bVk8P;V|6yzT}&WUmxv6Skb&b^Fmb4| zZv^uikO3oHU?i&^9Mx#+ssUasD63@isuCxE`VF1~bwa}iAp=?eIG}Dp>Ah6}J5;65 zhUeFD5keA@62p4809&CVk%PpmgCZ$O>^$`A!UbrCzD)uxtK!6~2E36?tK$4q{ny38 zA#My*Eu&9J;M-TF_l6)ct*k3h-@ z0!9h(HA1%4WqH0h@1^3L*K(85CKlAI3r2a_Uk3T2gpEVO+yi9q!o8f0e%Ze&p!0IP z`LcgSeCy?U0%ZTPKoaD@Lkqp_Zqy<|jk^#ql!5{?#()_{9-}V+$-s3hMdxBd_AZC1 zMJ8_e8Sa&MP*R{dSZ)>%SvX|e_cIjd-h@mYYS7Q1$3RY-a&1cl?jD-@?AVCq$h=dW^1PF@hVh}>83l0=#O7_7A^#TJ$ zknM*K+DV++w=}(dFx5B^pwR$8EqI?eYveBs{u8v2h1l9swcwjqHRFY zFg*d1-7KMKh@RkpB7n~rc%ESI`9R##AT&>C`vY*#I9Q(Gv{yX)bRl^T2gkE@(9Nim zC=9`~1^iA}%^C&0V}Nx+Y-|_Uol%1}^1P0N+KJ#TJHzbkEz1)&aD1L;QL;pxVb42J z-#JsFf#Uz4S{fxys+wk+EapkZyYQQ7K@guM-7BB^)Uzp-kO0$dfHaWbhh}CQwYN2! z-(j4RbKsUNVf>!S@z8q6H?))r!Y3Sz?H*~LQgoCRI+m*Dn6|h)5_90xk2?MXSijbC}Wk54s>A*mNZW@C2oaO%PJ6GQ;90YY_dTM)FxLC#5wKkDWKCF6=?DX{=9VSvvioU}?s z)|sP9#hsG(dGFqi0xHpg7x$2KRx> z*> zmfD1Ll8aQ0b%0{iO=qoiSA*IXiqGR3i0OE{XYKK9P4=kaR8-s3ZCa)=p*qtTpm|f$ zTmsY!+^)3QTqiDdTZ`#unr(&l5}ybB!0GtCF@zrfJ@B^wB=3)gdCB{ukMKGi_k+A@V{0GyEe9UC?pl3+)R*rH z=_IT=BRm)y+p_OzK88*TE^<(hbEPCfkBp=gpy|j_?{jkDvPE5*Nz2V@vqWLC!;Uv$ zzuB79`CTp^ax1%l_AAQ1lQwj~xbiS;H!bcOp-|gr0ULy3BJ?5;ds7?^LMCms0mcBl zHsG0+g%USrSuomKI7flyEG?<-tka#VZ(oG$U(i_to(48#E#TjS!0JSTN+}Xf!_jas zS7!3TTF!bXdlYL5hnjubW3D`R!=Xi?J$li}Hl z7Bw4TGC*{ssJV}{$Nh+!W^^Bg#9yJ1=9xY7&ri)wY__lmO)+c+<`uMH8$x(N=ND5GnNt}TcMvu*1Hm>i?U zdho zzoOIC*3%YtLw$Y#9M@S?<{8lub$Wzi!hYwXT+!+|Q76nqy)fdLQ4!0kXJ{in~0aq~)Rqmmr881j*>>Y+{1 z?nxL$c=!Y{XT;v8JQSj|C0V$6>^kxvKR4?la{H)tc;DEJ)Y0Q7J&mM2VhCb0bS7~z zA1JU!MWsZiJ%r+uw4~GKv>gXd(dn zNS8Vi*YV$>$q>H4;eH)2!r-u}0rro8(d{DEL2x~9#w3rJ`Zt>a%^f8pF6as2dBxm` z!f;79CL9MhH^M}=Ip_UUH5*o{;Hs#?jChn1zeC*fzB21$&w*}=5=xKzW23LY3=$#i~{o+ZovQBZyMj zK}PJ~<~E5Tg2~}PG;V$OW6yAmDG`gC!*qWNubDH7COYS%eND_^YZ_u~!Aq)M1R+%a zz?=!H8jdIZEGE^kooh8EHxMWwGL72|@ib>vc-Xc$aL!mNbt-~W+u^OK*mLrhJ?MSH zB4g(Wn$4O7u6Yd`Uu`g1uiO%uOP1V4SVRfri=f%pwncHKIpvvKASzq$x14cB%LUt3 zpdelD6QHKKQp|)!%M)5Y33?oegA|#FJmJ|dQEnjOV2ug`%(eW|h_cNMe-nDKfU6|9?hp&1;xE87`#FJ{?#h(-#3~73Ju7*UG4dZa;s1=y=ParIj9N;Q`aiU~ZPNXJm`{K1 zTRtJN58pbppWB6F;ntRXz;ySJiW(4T^OU^uBKxCN1dB@2ENc;wT4;ABHWJO9mW;d= z*J^6&jGd7y$Q5h8fphJxc=drLbu}n02ib}TSEvo*+~5dd|5`Xg%@Q>TLBl*u=!3Oa$_}m5!S01Bxx6q0f0g1mj6$u#fjC7=h!%Hz!J?{2ZQkUP|44?If~T4LCQE<`Sc4#iR2 zXON=e#fJ4XD?!(OR4kY$bw*250Cv6gJT%MnT1hYIpo-OIp53Rq=YPRUE)ZqbXSVNb zloKonWY(;h8@^*VwSy$+A?ijI>+|x|;pBG5PRSQ*lkDydJ9Z;zbQ|@y(Nokgfw?Fl zgYzU6#8;P`-Seb6os$2?nDQX$|-CeVwo1A z@*isbQBtx6>@nOuTkfz3@w3l1TZuwsVzIt1N=8$q{Z%D!fUN~4q(>wWgwQ=racco; zpova3u%&oFC~bJBSK28=DhXW)Eg4}nzG?L~au0-+UI(BtTOE4sK4Bx?DnaH@s6w|M zS~?|HBK0gzD-0qt>OePQ>|~wm2-}olQjp)-n%AotE5Ws{h6?Qiylg|fH1nK_73G6o z!x&rP82rt{AaCvib2B0^;xIUyqY*X@dS;OBIBd;$vv3ri=3$UD$6#oVutCM+W*!PL za|BkV#(In(x;0AXh@p!`#ykWjW;_~ZH1J#$4rYUI89^}u0keZ_8M6-tu`jp9w2Vfx z9Pjdv82sp>CIQsSNW98O3ujF0Bhe|x$GAq0^C7S(4}(HE9)B`~g)|y@GA>$iADEL7 zD=gxc@d%U6^nAtEaXR|Zo7OW%c&W4q*__y5;lRV%tEhwA3Hb1+0;5>3xRWiKu5&_| zwk<2-UTdC_urnd<>cY$^1Py|HIZwn&%iP#rQ0;ukgLiB_LB$X6Z>AN)@>^;_({Zop zjTQ97+P8Ku_w9CL&*9Ogn zVhmM#;@fGyo{m?&!5pgT?IFC(uSD9`h==EW5XFw1m=7~+mpUCx*$5o%5eH59AZ&3e zHNY{f>cjONdx+vu$8YfFPJxt^ElLJsgWC(Xn=cJ;#z>vBd#0v6b|yoFQA%$qCRY9nzu~}W-FkF zyHViHIk(8fjAo{0%<$QCvvIF&x&E`UAha6b+oN!>^996nphZD#!=G3wOg3CYv~#0} zt|S=)M?1~Q8=k+!`&$4eXYh9N+2kjYGm>PrtXMfbO=cM(a;jQ5Pf=2U5w<|Nz*k$0 zV34A=Sy4%zoj*H&`dn!s=sTg3o%PI&diT>Q`FW6{>d8~`GyMBZ|N9yFnfy%16}ckr zzv>_IGnuGC)?M?B=F_t_BTZfJIci^PSy1Dj9iJ-+P_KQxCj4@B=3*f{Nnu)cyz2nqXD~{ ztZH9vYqKp1R;o!vBM_ zATy4V8Cy~)I}at++k+_Oy8S>s}5fS#kbsccWv6TLy;#*m8>+WB3VKU#BEOKt)w z+Lz|-JIEy%tHxxj%Rpk1r*7v8W_kK$=L&(ie7-hfkA5`o$X2G|>UyJ~g~rDy2Zph=d{nK)xe9v=uu?fg3&~HxxTNCp}7C zOL_Bc&ux{u{hZ0UNZG}F&U0R^uSCX^xQHeIYHL5|oY711Kpq$lJ1o7y{$GL5*=@wR z`opd*IH&AUr=?bNv~s*D~b-rgsiz z2{G?rop`=2b!*$mGTq+rOWzNliQvny2d#VDz!}5QfZ7gDQoq8mc?4>;0;Zk)G|AXK z%j`xlwIZBF?%Tv_g5*1pB#EO%Tvio1)t(r-Hh#GZvd5Y<;K#T&hbNHnx;*yOPAOB0uK)>@~-)`<27 zF#{2?^oL2l&$nIX!`lX7V76K3!=@;Vy85W=+^nUqA|qFCn){8B=#Ihp3$ulW*eUcu z`m28eK$xi@t7R}QZGZw#SzhskX0DXlz=j15P*c@j2`4rbhl))>VSWTYGO1;IW z*iomW7nu}iirpB(o5naS%LJc?wHi=X^r6UDlV@c;UwYm<2fq)=t;<-agiROVQtvFS zqjo)qrI1xlSL|q5VT4gtD-U;~u{!D!)XwPS=@}$JG%xgN3`h3Kc8ioad31(v{YfL8 zIdAm{qpj9lR(+P6ZTItreJakFzNxmgPcDcxqk5kXdp#82$FfYRU@|{Gw$>}1rr3!f zRN_!eFC3(NV#Hrku`JVBk47yA^|KO<{>_1M+cGyiA3X$JV%mXea!UbS4$d20&cC1ynq0P2>@l)fnF7twWp7fxjVy+MKX^ zR<21QB<5uTi>FfZMZWU+_U9%p4KzsTol&-qw6=8wRvgh{&-=6oI7WQG-72K`lh4kd zoucP#VICQna^2b-})QTL3=mTmqr*ARFQqD*zHwVuR5vI$fIG@jj+Id5V6IXm`2 z0J{N~0E!z zsf5m~IWh8mKtrCfGPTd@aAf+o&aLQ~0Dd>;i@G%SZYP|ob$`zo=^lu>Qt}%VYM9sA zHLFls(jY&rmp1wJN`LF2iS-^?o}x(#7kP_>mPl!D#%G^QV^Mb(EQd^y)Ryd1%7mDk zs;<`sOZ1d)9g*)e)p4j(vQppZS{zj01$=WrYNSn{wu!ul*Im(S86~x-8+BuheOxEd zszcRkr5YnC09FdP3=mj9CFD&lD-bF)Npk#r5JfL2l3}MxQP2gf)~LR9A8JDVQ6O9C z-B1Dy9o*%M7cyZ{N{Uts)EO!rETH)^T@zZ0I@bXZ9x-0$G`;CG^sR&WtMy`=P^ENZ zU(CSM>C-1$4|DE;2I!>kat*|7@+=s6t8@F!8sEAVN^1Bz&*GHnH#(C;Jv1>gS5?;R z48M9PdBq~q)=uU$lUNA2DeHRL&O?M-{N0)fk+A_uQP#7q_od!|un3!NX_w&fu;A0$ zx~(nhcJ1hhoZy|ul9WPJr&~{-?&hlxMO@+!K}(?rDkgrmdDK$-ka_d-3jvw-;|-ohpvv5yEwEn>=*4<<4um z6EwVZI}>Nxt!4mtlj(=zbgNnkquH9U4`3d<-*%q{gfw#s_5P~(%pN5#tU?(j0YwQv znqa@69xeBDMIKkc>7#2z9`p8JKzM1V+^AVfohofqBQ`*GUB4|rbrEWTqANn_`VJ&V zZtMWt$TjpqzRIYqc=AOik~`J+NW)CuldYcShJ5$|-dY40rSWJ4_T;2pACfmb}_Z!Rhys4XP zN-{@6UcpGOMNJ+k`VxSrOvw!|o|9L(tT8^M>b%BJQ@2dkg|2a-9R@IK8V)UAn5=E` z3QZ<~q>>}{q58wo89Dkem+GIqn#-dpxm*<)Pk5zdlH944&h!IbntbjGA_rL%-uK^p z{qpPQASR~*ftuO9Qwh%iWHqy^j%hE&M68NZthj@}0-MB#dli+g14e$IRzNn^sC~pN zQ|F78@Zoj8sHF&!7po zQv7LL?S4{8j-DP($+eM$&fI?6`@9tAqO=VGGyjt#aw1A-Q9^T)F{%OIG^1H#62Yk# z>?|e-gG2>lkO*ypY%C7*gn>pVYwdHMi!xyb<`{1Aj*etig#d^xGsZ_{@FYqzAy$gI zd>KDZ<_Sdi$xB*MlyQIq`K-|PKvd$(4B!{w5l2YbL7qr} z?JwSlbc9ghJY-n*-&6}<;P+4u|2cUpa^_Fre#dA^)XqJ_kKdKNVqR_(bp)v;9>P^@ zb;vwmI`h^6MoiUAsT=fJ<2HtvGPd8-TNa@l`_0@;z$B-*`9rHV$d(+F?yI9?=R3XHab%Ii^ zmsBxQvZ}L+7lFE_u;kSOg9vO#m%hHm%txyiT=u0U9>y~N&tpCQ%Um7v=p zg+(5wc!CDW+&)I{DbMJPXS`Y~`QUVFq(e*thGE#58;uruti-cn?Ib7aw{x(fo=$;h z2IU?=`0fZy=D|UsXh-=ad=?vy>%HNdcb7Yz)rynEpVjK_-m@BhxMt-&Pgp=<(4M%% z9oO40ru5gkn3U+Al&l(cqBoD2^U$Il<4ZOOo+9^WZi{p@)f_SE7tZ4LdeZsM&pI=7PL&zNI7)S;mr}RY1&fAS}rq z)lJ7D$?4s{f%2x+v~!$*^LMW`0pItOP`i1Jc!8lxl63?X|?J z1P0-}maLrOt3Q>&4IOmQ*mulZ*0gVE-?4e$$m*}H=hx40Py(ZMwZuq#u)Jzz=q*!ZCX|e-Pyz&li&iUM(lu2YL>r-Oqm;QyJel zIxf;wyI*U`=5>YvH*!j7!F9~u87RAFq~_^gLbWQZ|(e$lJDp~$y; zM3qL?D`_eq9;t*G_n%VEa;E8u=bsEW2+Np8WZ7DNK&6N+}jo)^>^9{V|~oN zWDp}gh}?Lg+dk;tz11E5)mr|TP4MlLB5lV1({UWPjdFei5j&Vs&UoTRndU-`MPe~7 z-<{`;d<5z5lhpg8&dnW$Gfu|aUJhyaav>m_Q)Al(MKN8@Pum?J*MXLF#VS@dNIHqg zb6sKN>w@Li6;1BEUXe%n;8!mn?BMV<%LWrnb!J)D>Vp-|bakY9go>Z72h^$it;1!5X+0pf` zv=|5-J|h{wV`MJ>T8g@8kccrH)=N{v(qSW-wJ2Hj5#x}8_Q}>>g1b8sq0W5QeGR3?*f&5T^HvmUl`=)Es#V`6bfgieXuWd-N`?=-SQO3byrl zIz2`fB1$4FTIxNozTanUkTETyem){ zUI$5m5fsOq$l)+&y!L!9md#PO#GI@Nu=8nCaI-+LUs)?~6oPlSxI^c|%BNFsThB|&m>^1>$EnE;~8ykg~?ChXL;FtB3DR;={Eotj`slk>Ku zd8$DM6IKEvVIL}1s<~*2{Y!H4g)d26gRTvF{-ewj@TT}cWw_n2X154^(`@V|b}^)) zS_{imtv5)=1Xj2fb*X(`ro0Fg2u!M>l-K5>l9X(rRFKkvphSp}ox^mu3yp4EF2m8t z-aeuDqh|D;r$)8`7k2|9SWv00iQp=iQY739#Go!`+EB#;>8Mr8(YhgHQv(Im!Sg!b zqsBTHDZ8p?8JElJ(MJEihcWods_Y?HMPj`TJ+^+q1QKyTNodAtSz%+!$r@LVU z9oX1J4{+javj_{d-ME{Ku@MSD9>6a>2rbZH zk<>~NLw>C0tch0$m+p*Mx)BKy(9Uhe+EhH`5421#t_JhR+9FwVzbllQuZjAe&Hz{$ za~M1huTAgRFX;vAGq6=cv9b+MmUl_?dbxb&J0b(8*cMJTw3K_89zX`w<0F$LTT%G0qlOXATu5Y+o8=v)LOR znfIw58pm7(ttwXL&&hwCy#M{rlhePPoP7Ue^6!8D{mJ_&eE8kzUrv8EfBgRR^yK9G zuipIi=IYho`RUK!=k@9i|N8mld-m$@kwi{U|MIU<&1*?~Jg+9AoZ!$u2dugXB4&WL z*)Po@Wto_j(hsoI$B5WL@YZ$<4lZHG#;<(5=#(HKaGAa2ktvMk?snor#a4wVX}Kn; zHjS-E;9ocZ7(0XhLHkl(?vlm0VkxgzBY-sEes@G30Hjuf-ifoO_`KnRgCHQt60wy9O&+h@FH#(@YboQVGyAHJr>G>SkpTaCxLH#lYfuW%jCZ1ga&Z z-iXp4m_;+`H+a*c$an(K_n@i5G}~o!1DL)appJYk)gl+UzQ;>d?-<*_2ZP!V;3&E< zeOrGI$pqJSBpXu@WRx29Qr4hs=pLZ_>!m2>;NG|5rnHFY8=EobS^RD^0M>RA>Rbn+ zeU$CqH~B z@F=u+YzD3vNCcVK+T4xGPiAb60V~71M$rQOTBpOLEMpa9urLeTkI^$LajzEV`<#3u z(a}2Wo|!)<^t6>7Ds`1;#!}3WajFBBbuQ1y?9_w^4iko&ndQo<)c{tclBod(9Gg7SzX=|tWRLfG-i(&bV+NYC@N;A5> zbhVZ&NxIHyICU4}>LeH1`vdJDlw4WpW%M__$S7Zt6Ui8PWtKGla_twF(HdE{W5t9m z6qO?un8TDqK6eekuSTG^k*9W9_5M7(97>48GPMO*o(dFCG?I4%REKw&goCEi zc0RPmLs3k+2HZRy57`Pa2`JWjMfR-#=;np3n5cwFFp8_GmA;~_n|*}xk_}L}{Th!% ztURprfFZp-Mt;9<*3S)F6&bDKV>yn#bFAi$_f_p_nx_+*sbzQd?Tb*PfZeIBP^_D# zDype(qy)85dz9b;V5=ERQ>W4lm{J~ctfs8~xcc_R8R1hlHGg?Y8Zzb>sKf$e>}|>i zEO_XGV{i7-RO4jm-u{Q`xO(!>lmBq49sK!Loj9esA)-Wxe!`9Gwq@FGuS9xXuw<&M z@!Xg{h$!Y;FY<^-8r84DgMAp$ff(ewT~tcXsKUAVMsi zPw2hirh=7X0XZ1#14E!^dA24iE-QM+rsP@)9B$|W&SXviqo1Xu&MWSZK2lHG_rox_ zLdxZw$U0GnVo|b*5r;yHzmK76(4a!-D0cOxCB?|DA`NyPi$3RiOLk+qq3SV{Ua(3w!I^lTDaNR!7j3zL z!w#k)NR?`1MZaQzlPHBgm(=MS>iifN9oB<;Bhom3UZ-?h^Kg@Vfyq4*%v*Rvl-pqi zRH|+@Te770+gq9&>2+c&%|MeT@Vd#<7?(&yM4`DxdfH<0Rn4V%)OEC5sDu1tpVh2h z$8FfiposER=3-p2py%XwEdfUh!&almgq#zj;!pL280@I_?J*8((s?0BX8l^KP8^DJ zCK}0KN#Z4#)J@8yTF|A)GFD=NcGUjkmO%@}1S+ne>+8&#f2^vK_tr|^yo`KCC89~1 zxCAhJnW#kb&N#f-WwR-86-Y^!IPqdfuF&lv1!9D24?v23|dbhEW z!5zB=P?I*V+(m9utX8%Ip<;VLS|!qjn*?WNU9dl>x1A9_Y@u~8N^+4;bh5s6-e|eo zL=dA*f#z|Nofg`lwJ!OKt4qzKPjZnX+rxX~1UIy#^(r?=#!4S1pFH-2bzqr{c*OHm zod<9%f+0iJNx~wxk~SN_h{B1P14Cxqv+qFL6|>edTDfW)bNNv(WnuP?Gxiy}mOH+9k z6lIMj_@#)8Xq?gH?tzx61b_otaje*OhEkg=VCBchaa**RW}bKDOt;e~&2Mqrv@4^Y zTG>}JTLY{aggBygC04ZJ30MI6-0`lhY?TTVOA)t4K@v z(2b_Hw!+o};VuJoGMgzK$*G-b*A&2FUQ50X4Ldv&>?2sA6bRpKjV7mFJi9c%M zecgN1#QUy)22I@IVOw|AN@s23qSNWEH!yiCDjlc*k~JmcY@lRAOQXO$ALx3Tvr6Mp z8K1yA^WMLfGyp>GURo`4HlE=M1~F@kR#yU#)Fxg7otEWPPJzoUtcI19%Nt8*Uy(_6P}BHVb(RFhxo1V6h0`Sylr zhgF~3^>R+$u|(Xn5*NU$4+YI77x|ZzXR#h3vs4V) z-@U{fYUv{sN&I)2v@gMicKkxo{XJ8&W zI|NVW_{p_-g_}F11(IvtH%2BGD_&L1ggM)t0r4qmB)*<|X`TQM6pU+TA8f5nM*#p$ z`+V#K!O>B9}bXxhvXR7o!P`6;r@X_v(X^jREayEB(4lqzz68 zLlM2?>@d0#D+Y_UWY}1@nz}CDzKqwSMuahv4NIY(_r6X~p;u$hS1(V^)#dLsWJz z^E2{RRO+8skU=tXU8?J)VDhb~;Q5a-N!a}zrbzquIQZ2t%L|Y;Mz*7Ai;bnUlg}l& z%!w%V=x(tEbR)5O#=X#Z!$(r=X`h~OQMP7qZ%ws6@Ib21=7n^?ehLvJHZ25-`O%6k zA_A@2*NT-3K!;1B%3n3t$TJGBE!Xw9@OyKOF)!j3E`Zg(+bo)R33;rNI=<&12)%|x zr;nrAZZW5uk{dP{rH&OX6xaE`O1OXx^#4ZavAmR1azP}|7a4PD7_*-9x@fYhMM_Wl zvF7(QW0=oKb6fLG%s1rEcy573UX@5-bNi0&*!t+K%a@`l9$n@~4gP0~2U!^oVj7O% zw@CP6qBFI+BJeoT_K+e$NDU=m8PS*~d##yE_XE%({Q6zUX zR0HHjToTqOhR`i3vcupHl5#S3qZ3#l+-=Epwp53xlgJ#oE%lsj7GZYlZ#BtAQ8Y!V z6Q(!9lcD}sQ>7(bkFU9!FpZMQLSTq)?;wnjV3ttQ+Jx;@q^Qe6NL}}-yJzA!C%1^k zCOs6hmhA?C6jGhG97XY>jnSqZOaMHkeaoVMAfl+7P*POeY^@t?Dkhd6A7k8&wyDRi zRHaH)ZVXr@+8u#(_;6)wb4i9pxSe|G_AD6(CgWa!$2AjmuGhUm`PzL%RqqX^fNh@M z4NYQTTqEadoHSy9(F`2_<(!oz&fRR?&$*&rlM0424R1Qbx$BSN9FrZVBXlXbVz_N( zMJslO^$_QoDVq{~2(3iQ=Q@J$+Hr=SA(({5gK$BZqsW@!-JcT`|7{pv5`Y5#0tA+lQ^7(QN53pyknH? z3Cxb?hKP3sZ&mb!}R`Je%UKxb2&gwQoW&FHlac`+QS% zPqP6S8dPWDaDsdNi*~qJF1cJ;pC!+obD8M}ymWa>x|!-?nM?6NQb8UV1R^Op6*3EC zOvN0h_mGsAh*eRF6*p1xW=-r2z?;?Hl1S|t&gxuEZ&PXJ%NL}g<$_g;4$14)jCny# z_giCZbm`7xTf4K>i|q?<67tSagbXB*A?Kn@Oo*o`6^e!9)XaaY&&E2hr1#K{E9!~E z|J`?Q_YjOwcf zfv8a+JY{*MHh1Zl(@uceDS5>ak`HJ@d6w0T=MBDNGXWtlklI(q&@WkqpkV;|AgU#i zd{t)^&6%iWw)QTkk2wk9!!=_ca3jPWviA1&VGC(nJTVg%R2xiyN|SM$p4*exg66Rg zEYV;qMOH8PAT-uTt%u)~_O2y>eEX-;sf54l5wTr~`d_CQaMN3!o}{c`c>vE#R5hPxD5Dhc~gmDKz4M66aykhS#Ygyw$M6{R}tOBhIh z#lirVxDR9~cRUQ{k@hIGzK+u{O+q`C*>Bh7hn>lC$)R>wY+2PxKqYSr#R<$<0FAHNJPfZvZpFJ_Qu~>-BWLwfXF=` zlT+7T8E>7#$K+Fuwz7$PR^D^=aIQ1HCJ($?PIR*JIS7#FzlDE-)&^9nF49!zY$9H& zMrW8hC0@@Uwdah~JpD_^DmF^w2^uf9aTgxC{QDdLd*jLm>3cgC4gIASvK&jrur*`A zf}!`UK1~gjoq2&au6wv!dCyHECn@VZAD*-*(TrA9vMOYbYs+WNGm3k-dqn>P{wJ0a z8ekb3dzMW8ti`;%MP)8ZJ20l)ke~mW@LG7c8sm~B0rcaj)vsIh%{P}X#}1Qv#t4G& zxjj@79zSsN@E>be1p|nGJ2;xwZoLfdqD8_Qlj%z?@8ZgEZ9LZAT7P}bp#~sZl--}pls-L!Cd6dx zw);sXIeL0DCD(k#Gg@W{2xd{y%zb{7xF~HyAgJxh5jhbhv?!rD$(R8a(;Q}vT0uZ( z`&ev`?A`obq@zTaW4zxvQeE@X)yu|N6;Mn5`r_4;e4|wveJmqaQ~g(K8t(X*0~@t< zcd!t|==1(Fprh+=`W$|V0rT4v!kP)LsI6rxUcF<*V7OsNlERxi_FrD9FNEY!%9K`w zA~Q+qjQadWX#Jg%m#?ngy?SwR^Xlbu@>l%^?h4OSaw8gZu}9&tDSx`4h;7>$GGRjV zrj8<19-uk7%z0(2T>E9eBony~A5m1bwR*uJ+=u1*CF#|@i*A||J;l!E zG-}|x*Sog^cjJgw=n`ML4T)H-uo}(MLY~C|Tw2btTOY!lH8}NQ?uu1_=7U?d!(Ywq z9Szlfty9wdp$nka0KXblIa-(4k_Hb-A8F^dQRnYoUA%nrYPw2yVNonk3&C?gL_?>J zwt2*?9rJ}#$xGue`DG?Mj+H1xCKmlQm`4o6Z!s8oLMLj3cUNnwqvu=2b)MG4j#*T+iXX{QVpCgM6 zk_8hwmm0G+%hpXHKy4kN3#<%NYEMZ~@_U}Kh0hSUi5G+tfXkpD*g)!0<4MTe{N4-4y|_jo7ydJ7J1tnF4RbXMdvrc{#Q^OVT-D&zTG zQ^aU4G)N*N>IbuaOGf?q1>MtOrxma>_pe4-nZcl}_SxSIo6x}?t}=gg1icGL?7wi& zBUJkxcrKTTEku#IFIh3er|{U z848E!-G0O8TYjv7<(*br__6#nb<0NkvHbq4j}VH#<4FTM`oD|sis!uIv=Zegzd`FM zYeK1IC04k@z>LX*xD{`~n0GnfB-rGIf(`^@ewiz+Bm~^tj(@t?e&4)=ti2a#AIQ*T1*-9GI-mU2z55 zd4tpO+UCIcnzUypAgt3^cc1?2~F;=vqfNFRp-!B>(oOCm_jv_Uunjyp#WN^G-%vl0K?; zac6O6d~9(mh8VCZl$3Ri;kAmF>P-%dFTfDe=G2In(;j1Ef90{UCcOz1c(!y`9r zLFszxc2%A7`QQi~$lmGTPq&m*H7HOVuACmyHs|Ru`Gwq3s^>TwAhwFV* zLkJlZrH21`ww2CcGk`{Y*hnUhhOV4yTEpvme|SS(m??l?*@<}qg>liL_K|M#@xX9} z6q3!MBvhJHSBI~;S;~jt%sY=YIT+uzZ2P-lmZ*17diB2j=zL+GVb{#ooM9;{k#CV% z)AwN8X@3QGya)rfH7f$+m#r%a>I~~MQRK#{)OIQDut)!y((3nV(%*Hf7U_sYHg&TZ z^UIf7!Zev^QZF(RvC7EiN;~+8o?gD(dSi;RdJyFpkdF2vod_EJTQ`YxSR47Y_TDw@ zy-vNop*`rAh^^8(atr$XAlC2J*@B0a<>eY238mjx5pAqD_UR~ zc#?{T{DGFKr&x4UngcGx5G-ap(V=+;g6qJitNxhrN@G83glI?C_Pa{*#wn@Th+-M~ zEsk9*CnBk6W=dPK>VYx8H2LJo6SO@(`TXN!7o432P*JwMOi4PRUCr{LZ!Y(P|m2umo_)^SJ|U z%cbEA25^KgmZku?T4NaxAjyL{1?|EH^9I8#;XU>wfi9epjNdUbmr#1#pG#seluIU+ z$gk0>@kPz*lFSQ2v|ndrL`~bahe&T`Q_M(L*D`_R$O{6SA;QVmdgqm;iqoJ0WE#dp zGm$S~)&l+!;RYMA)MoLni-5-_zGhT{(b+d;Evql;Jk272V_UsFda*$PV|#UKhAC$q zg+Hw3=ynRoxK%^%wkc`5`LYPA+H|h=6U_;&D=`5WdJ}9tCGT?rxXNdMdmIB~t|h3= z_BgZeVkyU9-M-Iz=-Jk*?WY|@GgDP7wy8N?u`|VO4Zv|5El~1B&Pue$#1)OAotkYh zgkPZytm{L!1ymQgkC~-hF^Y`;)L0+al!&}Q9L79C>-qcXTgs_S_Ljv$1p>NSas z_1;H(7(4TEtD7znpGmmBAn`#F?VOw@7(}k~_2KZ=wnyaV`2`z1$k4!Ju*;!O& zQoz&z%u_4KF9Ho_zH^3f&`t)DRkdOLl}y^pA`_Iu*BpbsM?j2>v3o1xi}LOHwFuAp z;q+Z*$C^vAgoGxW6FEcq!Qf6R9=?$vH4t`$_r>Lvv3L#OpR!#Q#^U$Q^K8R2S6k&?hYIYR&8U79XKt73 z!8K_E?qLDNB?+RTZYd zQi5p3isrGu`{h55y%G+>5jY& zzN;E?c4Cx0s?h9xcc6AFeN8rVPQ~QG7tEO-1)mrrZLhpFX}C^BDhFMO7ff!q=eqgy}O%-4cJ>(jmA%JOj`)yXAjuU%eqBA34f* zO%W7M+eXv>Cvv(A(I&fRJI*LoD&F|KUMh?~uepQFe;x{;>2^$uH8yg8g)1A@8% zsNbN&?%{g%*D5B&70$o4Fe<;dAC9N?yl(1Bc$ zrPxYlpllJd?qrfy*?QFxXHdd6F@%>r^)a?0A;B(n?mRGc zuFmB_T$-)SO$vAhW|I8aG&m<5Bz8{$Y>xNxB*0P$fRqsa?T@2jXiig@@Ni=ran`c) zGuuldh988LxLCe_qncY|>YGFWg6`I4;!(}jHfCTj0b)Y0*MtJnJ1l5+LU%tkUa(^`@VF<1Fy;d^s=-{|(3I~RwrU-qWe@{qnLmMxG~J#C%XW>R)JxHaT# zF5eck5UFNz_P^3cCQCJTW8r)YSl=&`OQK7?%3>zSV~9T1?Z}#tIgRr;y)q8-?q%__ z_+X+={2?%B;Qg_kYFS-`lq$hiZngayt!Qxf_e~5xa}7RfFk?q^oc?}xWe&FyVF{%`cAF z(eoqC4|jb&Rk#1ql5i4I_S`AlA4g;s=GFGIlj9@nf%*^y1V}Q_@D<;FUG=3I2mg+PXs9@MiCg@?P<|3S`k_=PQmhj7@6~mR`C2kjA;Ka^$J%MDybn; zrD=5PJt%sa zOvU0G#bS!*62)~wA$1L(ASLaTB>C(rNh>~qq-zhG#DDLl?Sl2jp1S)W*=QZBh&Y42 zcC@Gv+2YtPl%#+I6#LLmLgYwsK}|yP8TKMJ#^`1-e|mN@xC6uzJlHkJu)YaQaglq8{~wE47;N8!V+(5;KpD1S|zL>vvDEnl{*W3zIFD)&2pT102^ ze9D2hL^8hYjyN+p+E*tbT0tYBOl~~#)CCn6J>NDXyD=dJ-J+TRm#MGm;8WaBCy<_$~K+Ec)I* z;L2qJ>hj)S>bci1#Lzfdh|+?doDV>6-!vZ|E&3C_av`OJ06D$6lL5pzK*6aV5Y@YZ zVk&AF`4dG$`Q|}!Q(cv$oI(8V59E=>H_vJFc-DC*!2y$K~upaQWirnsh^_Qs;I)Lp;&_nO7?21 zHmb>xW2UEf$rV+nJtsU2N2-OPaqpG-wptR&fbk=c)Qwthf^88-wsZDe61|s3(qRs8 zU~CcQ`cDx}W+{BiI)6!_dc&pIlA}<+xRYbpk2v#3$~UhbLqmCCs|ERWXHf=wn||s1 z3JP|w>bo}_CX=G|V0BE|Bym=7f&Uep&fL0nb`Mt96^q+!N<*6@>(DW^)nkn-6H21@ zPkCb+Ot9-~H7-t?E#&TuVowjK=sI{OB_nB>+b>uvYfstil!oWe4Pa!7t**4nIgK!h zD9Rl7lWMIclIwGhlLf^}AqhN^J*x9z`~OU73WnioIO`xME&3%7F7^`CJng@cj>XDD z+pH|s9Zy!97J@rb;E7R>{tCj~q}+h%`dCy`%o@*1?c-nkZyv)u7D;>B>I5s>TNs5n zg=pI*3ci;0mLf_S)np&HVe|j325w?}qAu=mwS@lZJEuJZ(`?rv=^usMrqtixM5XPH z$8V1#R?TUoWRq2^=UQECG8OT_&q*G2Bbj(}6dFB(#*f(RdTK7Vx3&zjGP07b=c%U7dwv{%cpO}Aogw=_-E01 z5b$*-7bp^{!Wdr#=Eo9t(TbHJZ9Yod7rQw5)MGPprRjIRYz9|OAjC4**BBLcM2PE( z!o31+XC*vaIo3GRg|`8%*#M5^XL|WLb0=_Xo|!5wIVA8*dGC6EaFO_hDsin_){TN1 znMXGg>1dZP*cNJgDX>KG>K}7Hol+viT+l&|8@{V~UZa`!1u7+E7cShFyEuO+QYvcb zc;!es@`uQ6nRrB+4u_L9F+e<33Qm@vEL9}!ZohE?p1)zzF2jcgZ3_iRX$l-X)sYVG zx(Mf~DlMpZTKCvF^_{2Y8q@}#K+n`!0OQLc*RkSTk973`6Q5L#x*<|60T?#-svIS$&3& z5BPHv-0AXh2WD0dSI{M8^7ZZ^rROE_UWV+t3FoMsvE$5>F?NHKgaUNq5^gMUSkk05 ziug89dz%DwY*61Ou-6-0o*QBHmpiiI=QIuO7^iR9wp70sl)`cb9mEXgNzOO?+mFtc zylkvYH{1>B$&;Cz2!YQhyz>|kn|Da<@TJ}@=S9(>IbUtdLGPnolV zlJ)LrBJbxejKX6=ikyDf2K0K=L_i8`kld6bKo-jsS~L7LU-8*kDdamI7o}3CsjPK6 zHF7345}F>y6Wx<>o~{S;jI@oLK3Boxl*p+8ylH00V*xj#0yl3NF(sv^PWuYcpv}Q!bzyyl zD4x8BV+Gh_3DNk~6XVdup9r�i4N{Yb@Fdf~qm+5J$jaS_4PR*LIGKQJ2evA#O?s zF9`%8?GHju4mX(eQdUxTh8d1P0~s<$x(<%O3Ko#)F^U*dXZu#5{ffQt$-qt`3M z*hv~|Mv?EisrP#j!|0PjX!PMphVEnZKnzqQIwxLJfO1{%K>+wlti%>4>?XopjiLPV zS<{k_&$dJ1MqNyPjZg8A?6TPIIj?uwXEf>eaku+9Um3bwYduR016(?IxP>8_5&>{~ z-L@aPS{gA0rZL=-w4uToTy6}WEu9Jt4OT|1DykN00B3ugY`0zyvK<{_Tn%Va_mZ-lHMKcxMVtSDCn}*)pQZP&Un!%FFq|T`;UmJmY}ws-JV`2TUR=-PFS69h^37 z-An|J!Y5(XLQCa~w_q#Pa}uDbI;7x}baRwO)bxJ&v)+5uxwXU{CxtZe4x}UTU+3c4 z$(FkzFYGC2rAK^GrI+ap9au!s6J?B{Es%>ZyZELhlf4IcYKEOD!JL*|Po?P%t$v?E z8jc;kA-T=T7_8$zOzbj@6LbVzAIGEdAuoBFw#~i2(PWoXX@^E&cN|V1;@EWgy<~dw znEF9>=CbJK#_S!v9|toPniTpxo!;xu{9TJ^y?fU;7BM*l7%m^S+#dc0YeFZb9DRZ% z6`r!QRkZq`VBi?mI2P*S)8`(2lDDpx;_T%47SFCQE3cuD(;{>5x|u_)1>mfk-ZEV{ z|1iw$pYYdUm^L4ijk(Q?a~D8^nUBrmm7>oHjZ9hsTb7f=PEH4)ReKOPRQtrip2lOx zj)C<6REeuqm{chk&#P)|t)-!!T7$*Ld&^BCETO54;T#d6lJQdswTjXq1dCeqt9s#gA4j^{%AcEB& zRTeTZ1d(U-8mJ7=Rm#+kva}l|bIO~dPuzYUxfKPJ*@yu$5>6Gu5KD_RoL;cWq^Zf* zEjPB}1>5Pvwr&$DqvgiNBDcxo$odDeFK#rLozDoiLJ>QE3I|FuAf2fLD9HJpRU^S6 zr{v?M=aaQl3`Z(Ss!Ej=n4DwwfY&fGUxdV> zKr(sV0FUl&QDI}(SoGeH(<~@P(I{08>lC*DsN+{9(xFv9XCj2PqtzAu7p$FBuYLn)JroNbw9_x;HbmwWz`Q5QLD*_P_ z_WL>akOg@UMwe36UG082UFyghShK)(0jfZNH1=nr$hg8o&$nYCl5XmJKYzju&S_Fn zsF@qr3wiEqi=ZmuMUnJ%kn;eOTsmp05Whg3${}6~sn=VKSG8(=7c^$p^S9O5qM1!{ z)KNa@o;-TDi|QI7M(n6I*Y${$2X7J=Ezj(2-yuGvuJaW?V@-UHih>=$I`7c?Om1g2 zO#>})PoCPoRG{_a{wE}ifBYcU@ufiPXUsLUTBQ9`wa>xe$PuoA?$YukhS%CQ#3HU* z+`k7brfmgns5CC|GLakDT@Tx@&MEQGC4&_ zO>O|a1r-JUm;gXx=@00l)%$s>U*gAwAzXc)14%}IWa}XO%(T2c@{SIKbklQ-_t33nf@kog7A9mvuEd|4d z3FJpnve^syBHNz1ye!KL$)4CTX*_rzhr9Eh!*y>}0>3jCJl`L2kJ^JyP%9bdqz@&J z`Uk$8QoU!+_u^tkVqB7?P??#)Od&_QPF91(!7%(eDER>i_(@=JGV0#NQ9hOf^6;a z*@1z^S|0bg#So5txoT>4C!oyaSh#xePz+Z#RAq+ZZ>2wvJhgOQ>5RXzn5!-d7%WC| zz+^cpa7Wgwj*9vSJ|xXG2#tOmOe8QZh7pL~ zpw1_ie5+pCO5CEhpgrp1G%6|C8iNxgv=k))a})x%bvN$aJ~1K<0GqtNpsnh1o(9mD z2@3UciM24#;Gw9^Oez$UQvzy;xqwj&o7FcmA(+FN^+?yMNA{vS5uq0SsV4sZeyg6g(>b zcCxpwcubg^u-VlLRn+~1H)pyTUp6;8UOl?HBkKSvgAI58y<3)LbV%}jYnDT6FCh#Y zD~Zbo0VP;=MU) z{e3ZM^lF)hCR@>O`eG)1eVtek<~gPMmUP4VZgFeF9@@iKvHF@by;*YY(p(ok1*Kfi z)q0aL7|>>y*pFUo-iG8Tkiw45KDoSvmAHU8MXa$vxlnGBuN3j%oVTcvb4!`ZiSp64eQ~r0bK9cyZ>(6$)jUoKQtT>m z2iqL7Vl5Kz4Jc^tvwd}{nJC>@c*kXPKAmd(-N^+k_7w3v8H_kIts`Tk@51ngJPbI^ zpWH)F+q(5tn5l-c1wR*v{@nh4&h&BX{$AG^OH4`Tl9@pLW}y(auyCz9r(7T6%H?u< zJiO!ohX4LR{`x#z!N1)umHBzTyyO2KpX2{Lz2oP7?LYGQ{v7G$a({ox&GG)CKN#cF z!RxeCo4?b^<<=*Xg`DzgEWB?$ej;~Y_4OrQwyh8d2_>(MDmpi^a%oml{G zDGkWsnX217={#RnAE3DOTIsBtmYWfQx0n|S#5S!S+}{|(!%CJY#pMyFML*<3rkYYz ztv}0rKRg6HLa=>%YuXV{u!n@j9V zOy1^?z#=-+0=yQ*;l4zPO&i=V8USjV%wpNWv>Plb7diRMgGL74r*+I>9Qzi9R-!ZG z_|kd^3}q>LuqcsPzs6>oSy54{)_rm%L%9s>w+p7IUtPxd){Cuzu=Rc+Qp2VxtggG% zR(&c{oY!zpY%J1h%{iFzktmWf7GVZcg^d8#om;oMTDxkR2DyKL2Qwo)%Tj(=jDBtl z@+ZY=f6}QX!MkopZ&uvG#@rfr5^V7lN$Ck_eI5kDfWqBm?UN%=orWt=+&{hBry%7C zsmLt#4oZ8DGdD9?ys$F^U?1|H+W=~m~-c+;bLik7m7~RrP1Uqoc)&BI3E(a=O#=5{Igtt zu>5}H9Dl7p)=!tnmS%9=`rf(ls|n^WNRC*fq6g(S`ZGQ~!}*&w)~*9pgi1$BP5m2G zwt7)1lG;Xb7VI=0x*oV+a{R*Atx9gfPVT_i52H;;ubPpHja4QxZkG9mVF=(EUXE4} zZZ69&i$n@j(b@UU;6e|tQ^aQ=_MpxSHlWfA#dTN?SGt9AbTFw~63N=Ndr?Rh8dcy$ zc-=k(>iy#`-mbO7wV)>%muGiaU}P8|?;~fO7>katF*(C@#?zQ>I`8ss<4l{q{g&C% zT~(EL-Fx-@-ws?K9Cv1-z>E1)OTKNEDHyizmq_JgIzm#d<);M5&o?_fHUwGRf?G(9!{O!zD_rna&a#iigp-OyQxJK>Yu>g1#8-XvpT4f~x>N=qEK}$J$IBnHyY?l&VPBS@ z`6UN4b>0YpH`V)4pEw}KY`qD3^--sqnC$=p`Zf-QH1baN@GhRRpev*O+OmNJt36^| zikf+Du=+b_6qKhlrje%VHsYx*+Hz8ZSFobQdSh9YKxlacsLA?8#6LGuRB*r;4Fw3q zcLMB`jcsbAa?JwqqQf_`+SYUgb})|FxAy&FCy2dJ3B3bLPbsPaf?+A=g$fzO6y_T1 zLOmR6ZLmrm8|I2cHK#G9q#7P|rmgE~J%@(Ipux! zelAR3>Y!OeYOrdv$4Mm0jgfPuz#5od$@O~qJ_H?BU&X+JV>uy^)s+BJuRgX1$9Ytm z(W%wmw;}RdV(Y+Ogu=OAGmwiskNL0m4=li+6xklvNuGYEx&kHM2qkWA&FjZC9!Dz_ zX^kErtdNEQLvib@PdP5Uy)W)-Jb12MF}sHbY;U!|xUMb6CguRww-^s&yRT6M?<2Mk z&m*?DuhKfPK2Kj{G#ml`9z-|lSF-^((D555-4kYN{??|m1j-5&4Sm^|e-_;xY^ee; zNHdsHF)k5YEO1!R`7gk>?Ac%vi`*ovUSjAyj@5H|S@I+(zfp-K&4%YrEnQ$1t}$}l zX{bf#vWwRKk9fhCu_x^cTZUVh?-UN}Jx*)o_gFe?UUJ$lb}o%WBUPat<{;p>b3B;; zc7A8V5_S=nv&n2_CjNzUl?MlufLlM9<&(f)_Va8!&VN9}U1gi0Qtbjd4jLV{a0^xG zwLzDa5+tuj8BK_#h{NDM24d;qsG?4fq$ZhCD??m#$RLd>W}8jR&UO}QpPFf1`odyF za{5V-r{qr)Rm`xq^`0>(DW!O#DmrcAPsDi()`UpSDdzy=<-anFyk--e6Kn5wfV`1J zN!H;pqEcz&40rc2MP14j5z!b30xM)X; zM}@&>8SS|oRrf0%FRA?01)-vhh~#+c;Mab>^X|}AiQ&|T6jqI;4mJV@g`I%BTJ);} zSUm=f1riZ~ORk@_t5qeeHS6W=~5VJ&wIoZ(bjs=*oUCa$xLuJF%MGSog?X*KN4l43KRD z)4{}QtC^lEmI}vL(2URzScx+lI2zYLlXR(s{ybInWIHFh%wXsMbLyNrk02{Hh9wGR z3)K6vfHg{}awIf7M9)X3T<9IRB_Qz7_JL);>^u$(DtOt1Yt-24>f#$!~YS7Du!8Z1`ykllW;1%lPRG&jNIZ-^@(yg3m;D)UtZe z+i$bRKb6%oHZ&2Xu2qJ8EGZ4o)@g3qGRH94)d*{v42q$;H>B@jd4b4CdzwUHTIOp^ zz!oimaKPXhl1!d>#i)UFP?RZhMQ;GvRea-{J+{xRSTA)4s6p(3Y8Y-HbnK9#Fb7;< z77kz(GS9KJOYh_dVvUnF>i=%FWLRH<(fM%RFPQEuSJq z>g*P!-2w&ww$!IsN8z~esD0mf)qv-C)k#OA1)wp6I5VS%=U`|LGB z71>##sx2IqRPe(pFwC~cYhrOJD@8FdtGjM7;3Bam=(O1=nm&Vyvsx|jeF%)}HzIOa z8GbWNA=a$L)`x1l3F`b=+a|2VUwU(p63%cR#33O1@@{ISj%T39WO-3f98dKGM+mRTa!NlJ zkAFxQMHUICimmrVE|xQ5H;~Binxs{Bz{#%=x7JULGsMx^I;K}jZUR)Wl#Yp8XCr|DaK-GX2B3 zK`IerC3Z3F|R*tuOKy<5XH~q+Vt6Qa!#RMycn{Ez4u6F8g z$E?aN)|_;DWKmzWg9r)Ugq0+-2X;Hb-9U8Fvf-g>xxc+~*!na^xz?5t6XE^mHHL>U z9V7A}Y{a|WeEEH7Vlf^*f`tD#}b z*V8l~!NeGBe3R84o!jF28NsbeK!D#8^JFN6{$(R_hk@KTr5@!{X{RZ1tI*bmWYTi2 z8tX=lg14F{yR55p?**?GxXG);H>Bhk-j7$Y5i``oN8QM!?sP8q=Mz6p77(ajPE>~S zY}jEjfrnRq zJQXU3+CS`+S;0_rauqNhc|t*>RURO!{A}HeOI?G%~r z&i5r>6N#Vs)`R~;V9Lk8akA%?KhfuGzh^Xl{Qojf6GxEN^Mofv%XZ@h)8YLe%d7u= znw)z6_uT5n6QaczqUC9WnPXEfkD!u`RF&}VkI`XZB|CeoSxopQ0&g4F!hqx6gYNfd z-Kw8uA|=s?(!>T%Z;09J@_QrDhxzLyX*3lfeg&Ak|EIw{)L<8ce7bZ?s?T-z!#=Dw z7-BOp>oBAyy`12x=EGF*7@3l#u^cf@qSie%@$#B+Y#KfNH_O0|V>!h(Ul$sumB+g} zmDx_~Pm9e=u?{$C!YQ2Fr2jSLMvF1te6}3BL05w{Jg5G{^XlV%2qw{%oZ*{7o<*Ot zl+K!j5Le$16jTL;-&N_|aUDjCAa=F7P~9zHUfqQn1GFXjznBk*o1x|!!|(Ie54 zqYS~gdo+vEFg0E0BzeOe$IrU&ay=!4rl}Ztz{0?5$M){V&ZJH~Vc`5=;C9>bf`)o@ ze2nF`FB#akdVEc~DQMZVVXF|0g`@-57}ZRH(HT}uMfT}XBlEt0uX;Y+wpv^0H}W3U z^eHsd6ss^NJeYZYK6122R28@13alt5t|c7_VIndFM*h10vA>!)eeBx}f%UH=_-oJX zcoll48AFngVdqmHGR}wZq)twxWjH2DaU$0RF~JzBFp{ldBOkJ+oa#uS)fof9ixe3i+JS z{4U*_ofLn^?sxQPXvdC6{-MBus~ckx^1w^WcK3}|S{pyD^2!y3qT#xTEDwpjqiWJa zk188c)Y$q!WcPB@Wt#}7=5M<2;$Mrl>oBxxsczEwxmTIoh2Y9cPj@iCa%9@_>wNHw;HD|M^h(FnFX7~R133BkOsE! zDKFTM*YP9$;pS+*?iwDgYgkbaF)EKUav0w8*oudD)fMq|quw2bUZeM}wlFq5@5KFD ztsY!4^OdBtv8m*88_J^@!RPGozkVd`2axcFCj&N@@1&+DH>;k7(bVmWXDbc2ttQ~D zd3l1Q_!Ep4pdGC3l^K>r`)=UIlL+V)ko@B{ZwUy_AZ_|1GJ5aP`W07spb~Le*ZiFM zt-xB$(6+@&9}}|R!ZP7MMM(%FENXn+wm2Y2!-^{H;5Tg{(QAy$)|o1#nYwITYl7UH zMo(tWpNwSFuC9FqcFXHe)*t0rjCOoq=ANuPT>!uG6ObtL@7cdUU#(Iom(V?#`Dn|t zW}PSakN>85?f{NEpG46#0^~)C8WQ5RBp%q`1ynztT-mMO?WOK#R;xoeoWl5ho*)Dx z=qBR!aS%)CYM-dc*HvoOw@xeBEj?CLF;6~*>obYWxxl_2KL-|io&Ic$9Si`cIp^u- zs%xDCeI_moy7h>PdM6-(^>l)pMa=*@NT7*Odud)otv+idY0unsH$C>6`1wq4Lv9o) zHLxWc(4!8JWY&?}ZSdJ^&8Md+8{7E>u0{@KjX{L-u40Z0VhTWieD36_N#GoOemd73 z*e~rmq~}1&&)f+1g;ja*l?LyzaPJ;eZtv?!dWDEx49O2xNk)O(Kh}&h+qUT``5Rjo zQmn<2lSNzjWUC%p8h5iZqimUfjmvg3^t-nGY@3yR3cDR2WnWr5>7f}wof?X40+c%W zo|?!e$^DVkKzJdjh|qzKGN;~Eo@peiH%mfxingD#P<3v;{RrOgY(kYWAa2%K=oxa@ zp1p$|qhG&86HNaWe)DmZ2+GcIBqq@`@PAf!>uW?V`&qOp6L>>fR+W|xYo-cXIEaC>&(OEh4cz+C;V zM9UYsd~gkj>(puFuuU)4Izw0TR92G4Y13xYqVTnNi0dyY!tyo7?C zv#pS3R)o!Xxp$yNeJS-NRA3g7e_QkgE3t-JuZALI+IRj5^k3-KV{w-$xUfs3D|c+B z+yN@}m2G7A3+FWoTBeqgt0i11&kX}b8jB-|D-n)RPMv|2%t(z~Qhceee3HX(69-bm zVaP8z{>JpKoGC0pSH-y6eyE|ZvW@#-!4>s0Qg>;n+bV2iBbZsaMc+C(*&Y9oQo zu}31;BPlgr=t3VHRJvjbgNn^iwR>cn3_q#pLb^W8LMpGHS<1#&c7@plqqaNjRD z=nnN-13It{l?YqT!f0nkniY$Fj8rVDE^@awc0zqRD0W}9GO%joffnC9zBPz1nX1rX zam;h2K_s3%_ILq}sIG->#Z>~4fV1?XCedZL3&lLjN{5zAe}Z(x`$dDMoK^ESD+H*_ zcWW-8K{>)|w4eB-SC%#7IRjF}PZT(sb%yd`zl{R>MEidcyL-!awon_1p}C#%_lp6FsV%VdH6V4lc@S{F|49BL{)d1ADK_0U4KN{ zB5k5i$}u=mA6wCS$~#n<*L$q((pN=FKh67sS4Q2rA%`S9#3k>>Bs{c@$xrHu*Tg>? zXkG4fg}vI};+EgAcjmxoaOwEV?v(Qh8A-QSquw(d`isl zipL1{il>b22C0>%4)Eg>pTwJx(AWDiHg-cgAVKjZT#)K0L5#OS{P!!6kTC8hFfJbN|rye@tNu#wicxd0XxmX`)?G zv2t9v``S&wp$?^LJTPPuF*6sT52^-#%Z?$s30}*ZzcVqzxxe{)LI0diA5l9~$vSyL z;VjDQOg;Y*-Os_nqS641o?qQ-dOFxh(-8p7_>k#**W`8MKdOhU)OV?CeCKLI_h;@B zd(i~5L96*N)LFp0p627P*+}o}_}apx%tfNbcvz!EsCUo9^PGkC{Nu-CX?Rfe^T57I z>SXkEjC@ZIaL=HW;te{Uh7a-LuW<5W<}1Ry6nyYSS9>CnfL$ zHSh!H)cf^wD5-BdY+v5(>PDNx7PCGM+U19({?hf8(^}K1*ScJhH4siTj8+;@Lui}4 z8}ipiueDp`xE2G%*?oOo{fW1aeJCgPQkoIduK30YW zCT+?yZ)eVyyFa5w!Y7&|u2@vNF0wN~c<$WGSrN;3Hn` z^(W=gwLFUrDG)vY;Yyj}HjW<@NIxcjk7{h4qCFk`1p&K_q%PfxV{~MMdY^}o&`4X` z=mei%8;3IJt**Y`hcq*cs>Q*5^XtdPM3=FY>r6P~vJX*2*iI3ZT@q4Chp{wba7iAOQM3~o+OzceYa+XU^PNNhhU@3bwQSUWHcX6s zg9=4vV{FU`x!Jv8@&gHyw1L3ngK}KMpb|A*E1duvCMb~sW)-w{)(vbbdII&FYzRfN z=1hWG&>8jBJm(0A#T&4!cqQD&v^PP=Hf?Wu#tG88T=DPTrm$p2|C%AaLF;bz2G<$j zs>{8863~;{89fdBcCRw+u06M%YuWUp+PM+!emx)DJR8`2m-hSPU-n&F_inqFA4y*9 zu8TGB{M>|;(G&k!Vl+Z8h@z+e+UePNWUkZVVg&W?{qn-Lhdvd6J9QCj;xg9Qd8?tJ z<2Ut8>e5g%-6h*ssi`Prd`fzG0C$ZOkn}P&!~J(v^gBobIh=07O^N5-{`u`?JXTq` zHvoC43Y8^_)L}o+!6DUV5{de!L__aRUSV6+Boe{lDhZbKSaM9=NPz>ICdM<`w?#NY zG@N!kw#^hh+Uyh!NPux7XS;+hHUBwv&`r}Dk(i*uRI>|Bm4fN6*2~J;&31+DV=^l~ z3Msf66I~12P0tO=RA`Gr5Bo6ro1{E-mNK-dnswj*U7HHUlm}$%N*)CgyP+TL4oz6e ze;5#`Inu5ug?-BCe=1t>p6? z@KJGl!|xUfo9na!S>k zm@@pklTPL77Y7}{DPSw@Vt^k`8rpC

6$pCS73+aWbJ|0LFN5SznY7b26tQ7v^|g zSuWJkqLNIAg9VUOu$6f+As-uqy+Ek-?{qW<*U2u>c>-%I>}U_fAk@iuD|`arK$P8d z7|P~0785qU??A+wR@TuBVH`1jeXgzN9V=&Vqe8h7ubqbx>l^pSi>;82F&+45bhd+k z0CnsL3RX`|_(uM85$qUYL;;m!n;?z6+vXT^_!I>+Ak-;#D|~U)Z9)JlP`HJ<jW}sA~hFq61+ei~2>5aOKem<7E+)XOFs8D(6 zba%ILJ|b#DeFcp9=js*-ld2O2fnnFBRheh}AE|!tk2jJ1pr|MN z-v!PVPc+52%JA^K`k#45=R)vG=yHfqS?*|~Y47Ea5m(vQNuXt8a*~Kb!p#$)^>U-g zrt0W##~lfD;FC{R@|G8&NYMD6fj>paUe1PbvR4}zIcwi@F|t=1LH|Wwqi4f-Icxvj zuW<_zJe&<;WH0|^pALpWkXHO11ZFo{j^{G#jzsQ{MJw3V6xg)p8}DK%6TDL9PRDb# zS*DU|V}FjiD^)aWYCiWCv)Bi*K85@E*la9q$Q)Z3F#pjfu^_g#R|ela@?d8w>wAi? zsB16{_VS@Ed-XwI#H6r{O=cRI%p@R0r?9xyR7(g;i_DP87njhgU<@D_)WM$O_AolHKxFF69Li}0Dn$FnS+ zs!r~a8IJN}rwDqv4@w+jb4S^A`v0?1j<44xF*CC~-}^JUINsmOG`(z}pJQ~r-0$}%6@3p@WG6n%8PdF!=8r})W&gL1 z786O|{(xN>exDJ^G~pMK$!lDqGUWc0Wg-a*zlc#4)~gCVg#>#Zk)};=$&r(cHARO{ z1lCuSDl`R}QXu}>^6~Ux^?AyuA-Y&$vKj44oy*L58)uYiZfTvRc@s~9;32UW;}zY8 zU*`;Py--g8MK9@Q5WvlX^JBnP_BftHr^P2OQuJAUUNu{m?DbHE$hM&yLZ~D`uT;gK zlm45wM`D6dT>tzb*7`eO^fah4v7|l4GP5*K8k6ohiGnESu3D}BUPBWx(OVl5s|g{< zf*brp!qGKo`Hms;94dE>{Ze<*?`!^L8^z@8-9`zUi6kLo4+fGkkvrG=W8M3?+`ai` z%R@Z~|ndF)8o4e2Ba+|t%*L;61kK+hrkOBFt=^&&gOxZn|#g8syUqO|%%~_6b zC74rx1PR&Y8#xuv1cPTti*TrOX6f3`4M-nT`Bv-5-?OcyjI^8bK52O-%08B|#tQ=a zDzVEt4bIWi`MFznIb6Mtg_Ma@sAAmg^(H6r7-d7CRvf%gO|yLJC9q^sf#%`xTD_c* zePK;nt{pSSCyiTL>o9230>aA^%48omQgViMEL$$F8CRUaUlAr}ulrBAO;Q>P$5T`O zjH3>*emo72I-NsT_60(z2Au*ab1@fzs>IHbc(+husAMVWYdEWt(;it!t`KvjI5P|o zYAA{zsX#bJI&lV6Fd@G*O7gjulfiQtuhC4PtB0O~FfY|le6)NWSQh1weqS7Xln#3O zU|es9`P7;CsEOsXMK^M2<7!sfUI(&SR(e90Q=3xNu&mp%#h=IKCeMgRYVQJ^BWW=` z1J#JV0!`dL*8eZY{V6!p@9P3ZW83W5wrx9kW81c!ys_C)$F|e4I<{@w$*J%6-{;~~ zor`_*%rQ0UxmY!-)~vY%k%e!Xb5G;q_(D22=L5XtaGm$XcDy-s{{m_@ovU!Q_c@LR zzIwUJARJtGBs(m6((<)7ntW2W315_$oI|vESxV|VBELp5WMjCo^{s7c%<@MT7}4d> zAaV5jMFI41DfYZK@{Y?NUH}<^$mt<*aRvj$6xj6jWd*a&OwrtRycdWfM(b>}h+|v> zY3xn4IS{D}C&{arylt7$yh~!2Uql-OA&aJovb@E(8e0lTbMh#DkuiQGPO@eTSS&4mL}j)%3da5+e&mMzXU$1svCN5TT z-hnR6g2=DjbN@%HtGOJ^RqQ@&TTkTl6{VvWl{emH_qWn+J=-WHxS9Tf&ER!~HjgO+ z=V3K(Krf6(XmgMWZP6HzKri{AFd7C}cG&GHOSlk%L@z!No6qy+J9gYnwZnG#!HC}` z5U#4Taumsg4b154^%6@SFl;C+Ui%e9s>#Sn1SV_pQ&3hjZowqol-FvXG87|wAx{4i zy-3TbN$++Xi_Ru5Za9I?PM3PjT>#6_VL`D<0O9AX&_TmHhI^+asB5ii3ae}{D`{T^ zqH5iFoaTECYTYF8RK`IPLWC6pav|9WIGgFAbZbz*hnS6y4**p@l6u2AX+=rvFm~;Ypq_dGHvd zH$Oa|*L1PIK9uWw2;{&dx6mj+b0I{>K&^@*6oyK&LuLjL=^x=Lls9t6OwXbc(`ZDZ zF|LK3jDGKSyAH)9J9tr+QZT&OzeZtK+EQV$MXs7~YB>H2hSUNbobajgQK5p78%Kx3 zDuR(`DUJ@Xn0(R3b_W=5PXFm#96e6HD4_(Sq&C4DD0F+Tc?IEy99aZkp zyI0wsM2z3!dZU8B0wl>ak~GWHV{2_Py3wXw0U8gxOyL0X&=Ki#Z6&KFM!%|#*E>d+ zrU^kBDjRJlyYks5Pl_CuWYk4HIr1eiw_4!w0&wmiy{4?1I`j9tPUj8#YB+A%(YnQy zJ5(^+`cQ5_5}%TLf?c}73pn9lJeFX;HHVjBI1jWP{dXx0LU*AxG@yl>2X>>Mu(@;i zaY|i9Fka>NWf&R_4j@q%lri@~iFKc14s203aEF`N**x0q#46EqP<4w}ftj+y*W)5w z#GN+IUw7>qA;gb}0Jm*u+_Cj^NnESc;Qv0nqtxJWth3bMg(^^RLcf|OK!ZGGY&i!3 z^-#DA0qh5i^ggZ4HcCiBIPa=hn1gbZv(vg+ADAEC+`r zO_}Xfn`C^5*!ZrR9B#_gb$(m!8W>Q@`O2M$4mzuNLc2q@FGoJXt!DOMs=!o5bY#_z zMI*9FBl2*NRQ<}Fo=hX1u67hLU1WDz{x#+y;IRxFMoe7`+nvK>c$TGFN}#>GKrh~HKIV67gE2m{LPTkD>To~>(VH<*iGl1xqHhL*st~tK=@}h^s0(!__<4WMo zL$3z(dYilSL#q}RPgxId_B@JMhW_N!EwR!3q!#jA5uzTc9-wUMnY_501LnEvGP`Z7 zeE1bYt0jWrJ&X7se*OcHiSx-7WF6$1cm)D#+{D8C6=!E8cDX-30Fg^FZ(B$*Z<|dr zFJSRZ_u-fOyGj|6`+NM|P3=NLK%5E`?I<<4_Xst(1fx}I@Fk9UfeI9!Iq^=)O(epj z;wK)|~51ad`w<7xuDN zCO=mR2W&rIqMW_?+;ieGjjgU_;@c~~3}PWBrH8@?ec@Pl1P|;`7_0steUt_fKLznW zTWL%AChLbJgM;Pl!)lKbp#22N<$xq}s^dB_lyOAQB_k!=$dXkYT=#MjV{B}En;R?q zs0yc}e~Jw~2veucO75@1k*6CYQh0B#!f(CkEuB~_@}Wb{tyLu1jDU`T=PFR*NI>VT z;xZ%q;(qwLRFRfBWL)W;EsFFabJ~>|h*(!;NEgn4vM>IGDYmX=LomKdTj7)L2o#w8 z>RfA$DfpdgLIls1KEb>)IIay*Qp@1!WS==Be*5c0`!Pq>l@nxZoCIRins5~1!v#y; zm2Ks+pT_|*_XXZpYuQ8*JIbNrGgWgeBHUH{%Amc9RlNWlVxvsqo(B#4CTF+@fCom@ zxu$iiT+8;?G@&Q=g8lfE{Hin|U9N5ALtwKLDE4XKeg|Kvs13uwYTN~LZbR8~hgH3g zq&jJwl5H+(soiDqQq+^(#=Rwfi&xKK@@fQyj%_I#;bziQzf2CJgcyIj8T81Xc8O3~ zT%>h3qQ5&n@OSnJjHq6jR$VFXobPvASGDEN?!9C=dj{7haVvQch^vKZ>t2M1ihTx4 z3#<{yrr-_N2;nycd2Rde;uH$ew{Bb3$hv3~hf^9uKYwNG|N&t#<6Dx3e!FEU6mOO^vVA5NC+XV@VI=Whj z$K$u+Lswl%&C$`@lJMM0{#La;z_3Xq$i)Ix2zHR(K5Lh;ti=W}0pMz{&w$ zQ*}9g*%LF{np1dmO6|Nju@r%c0rHgIa6g-2QFXd~kP2buZ}~><4MkO!%Y06J@_%+1B;18@4TrTGLrULaGs3n<6>F&-P;417s40N(*I%hh`9x-52M_86Pfq zP1_}9g^Bsba=$?vbuTM{rP5ko4d~8iP%qB-D&cH^xq^i?pL2Cc_OQV>X1{z9IJTiP zH61ndLS^D^fzwR!&x6qDA38WoZF`eqHXAp|xasm+oIDj~Nz27kAYPtIG22Ak94-fM zv81KCD{#8}OCUhVQvJ^{f@bmsURm5S=hd%Y+=$J^N^m?^jImD6 z%_srJr=#p`!ZQ4}D~tsJ>d`+jJ&;>Oq5OcxejNjR$y?4ic+XD5_CEbuE%oqfs$EhETUx(qEi2z7d zm`?*ovZivPAS!7(8t&J9%^w8-eANC+-ouW;WMH$UbLrUo|5}>OT56S#Cx)}9g16Sw z6z3o}8`mM844<+@(L1=ogg_Qd!<)u~1Y#|)_|zMHdtPKcYEHk9s}uFzoIU_k)4+!8 zJi;j(kF-z*<>hJc23jv{-DRY&)vbtSM951}{sxIst9*?LfQ3(11Ij=9Uxt@cD{Mm} zt~z~!tsQlwqF|71-j3Xp@@Krr;+E>nGGTc=bDKEnEKkjL%7)&j6uk35&JDm$OL2g~ zeA84<+?#_!F1bxMMh5+-^skR_Bs#RwXr_&#Xl##+RTnfGMlaBeBJr@y+ZVi<4<)8W z7*J5o3Jla#p)@Bs=39PzTUnyiwkH%GUF4Ly7P;DGK8DbCri@h(Pe3<=(F>Z$*L6*U zW5Z<|NP4wH%~3f$EL(6!G7ZEj_uLPLSFBO_>i*3W@fl%M}l&Bq&)> zL<6tBk8(E{#Go^0!nibNlYBoR+5-fS$II(OTrUIU{Y`I>(dkeg_aZ{x5fya%@p!jm z@efE3>}@EcC%%eZc6wHSCn(gg5UJIDB65N+5z#s5@WRjtbO;`5Q|cIGNlW-84=xyX z%Ij{nQht~YrsGX~rC(v|=#JNM6v%W#`VGf-SwbSOC}zDc)xV9C4$RfPP_BE6M#vxw zK`+@22->AmUw8lILyw<(j{)ILQ#G|nmwF2rw7gU5hey246=!5R}pe!w50yAF@BV0X`_dCZtoUdofZ(wJ1$T8 zv)V{z1mhP^5;IF?h25 zjAt}Z!7x!fUAT(WOKUfgl(r>NEZWX7C)Qv>wiwC|Uf8G+#`e52KjVTs@SRnDm$Th3 zWV9KF(38n0H1Oa*2~XZqhTU_l7p9|XQIcwz(jk+#Hm@M9B( zd)csvVn2rfe)P;E`=%bBzSufXlZB%yarrV^~f9Pjsk%;H3T^#p!4HPi*W%{?nsj;L92=$N#ruM3MV{ zzqkrJ`tXma9$k@arL2uG-Z6@?xXz^eVdLWi;FK5G=5PY8tRx>#1TI+|}@R z>;E7S^15ggu*>KGguDhzNPxU<9Rch*&^xUNjpTFRIEqC!v`Z{{|n ziZE3suMa612*_cRBo)4qxYJ0*iyo@v%qSswLt#aG(DXt{!%=-p=r!|(44mWWqe39J z_tcUxRZ;J7RLn2mXwqb?NveCPo^s!0UFmYo_4#U6G^7{O4r0n8BhdtSq@vTmkJWeh zyi6qPg9N!f!wPw(UVtABjgqGL}VEbhT;7b+DB6SU4@R=VS zTtxy0zoH4^n)tcv`hv)1@U~;#Nk0G0qBB>#6qLE^=rAJw0+Y~!;Adx}w=NQL_YHBF z>fJoG?+}FG;Q60b-7ToJ;0FAiR0f!WiEVkv&Vi2<_XOyAg z1gq6R51v{)DGF2syKd)xP@g4lS=t+BoKDmL7#xx0!1rd8K6b+#$4|oyWNXbXigb{H z25OMgEOqf4$wV@-`>a8V{NM~MVhEfhlChp7%&AuB2@2Zl*QHY3v+vQ9bR?I(U)Rxi z`@d($@=ai&Pc2YJh)$?+zb+)~2?hIic)vX!nY!hdwyJ57jCYUOq32dm)(N2$YUH82 zNoo6pz-0%t__uhja^>S6x&GbO$7ijPfX%Gl7LC2vc~?eDG{+1_)NGCdi~}ElUUt34 z2IpLq;&(Ip>V@kH%gr>&)u&Ry@Tj>VXK2q_JI(0Lw0h4=_*2QJ>r~!Zr0Fd(o6kFK zFuB1}`{_=*jXA^`wIY2WK$Y%X&s8a$26FTXa^qy$PRhTs0fMhO4`IKNW#O*N#}b=>N$Q!e$oGi`FxgWFHr@>Kl353V%tcW+n?xaYlJEvJa%Uye^2@B=v(=~&iT`Ie;(8tyr7OkqafV~>jWiOQ3$$o zv4qK3H<59&i8C{2XDW+{Zxhix9RX;_e#$d4lA%rUw>YVB-{S9*Tt2Pl&NSjnt<<-V z8%|+g2QrhI>(1!g3*js*% z_@n^PN5P5j4ohuINNl>tDEukFct%CMRgHkmuYv8c6h&Saxs3+F#fqD)&x=P-QU+$K z7q%v)lBOS+RRFAECFJC(dEICUSTPCT8*FzYRj?zZE9~8?V#B(H#P`2DG1nFPM9J3P z+Nk(+*R_XyzHiy7!2Cj>y$oyXbvjX)E;2o@H+3L_|7aP2+EXLO3iAr&0A$#T!zv_= zg7?Rumhg?)jxCq1FKE)d(mQ0>#T5!9KxJ5=T(8HD1Mux# z0=8rQd&zPxK)B$Y8f)Fd!}oVLr7YbRlegIdIC+ah>E49%=Eisf4Td6H{2Z$oaeq+l z9M1|ih!7uY+XC6Bm@{e&KaypKfV9EG$2k|SPS4x)kW5&U7bk?|l?|w+JweoCu$`Q5 zC^J08F|p`w!)hOn(hcosevk+~{rHPNB%`dVFiUcK?>(|emmNQ*1W<8^CYTgz|w zvaqLmL!pWXy+&Lmu9Ziwdtj|qr2Krlrs2x(77Rd9Zd9}Y(RAY3APFppWb*eiZp2rz zIsW7{O_k~Rx)WWrU|zuY@Z&%7%H$P&#Ww;v2?$?c{P)G>)tr)L%FpZ3jF3Vg&o|un zTSjs5&9-3R$JIto8@6Lz{F&CMp}+s9%*?c#YS+9EZV~4n{W6X5vTt8a$lTZa^aeXa zd{-+f&JGpEhG+`yGd_q0a$T+(aC$cdVQg;gWuUT8t+ftEO_wIXdT%d#wJE$2!beuW z#dn#X?$Sin&GuRLp`}mjXEE?&Mi>3aa08xt)v|O46dTJ;R^3HsZfpUql;3{PQ&*R& zg3ySLwCibfyTpu%>T;%pUI=p_CzR?ag%LcN)qfpjQ9V7p9P|>YN&9XwA~ux+0W0h* zFsb>ukwKvd_NlsZ7KZ40U2BIJ#%9Ch*;Y~&qR053Y?sAFWHj~I1-^eEgtftNKA!b0O_xPz6OAQA4b64UzmNcn zpbLK&zTYbeF5$Qa?IHFp_vnDc(bJjJ)`1h1F7lRy)qn50&<))3S0L1fg|gd03&~;Hedj50S#%Wb*&perKz*w7sYrN zc_bo}u=topO@HEW#sGVEHf+cZie}jr9hTom6O7&Q~{q5_gC4g|wJ zNS?>VS>OB53PJbw7bik_5!sP6bh~+)cTjk^MRZteHcracj&F9S>Om`SkHTkEzP5r+{ppT{ih1Y*#? z7r)(hf7>&*=O=eWS-9hBi0>j0E*8qqg@1F(*g~(sV_egd*^(C6r3tj6laGPLvM=oO z>AlFxApgRknY3Z4Ep4&VU2tHG<4B~Yzd5Yf3rWGC%{6*RB7t;uvKSnX$DA*)bKr3UXT2fXWk|($Mr;qxkJ(yd9slBo0%Pz^ZK(M|v zK%h5=zaSQytM1$T>HdCLArbE}1W7p>$7VJ`rbnGr9?>WGS>-<$`%9fP2JdmEHBxp+ zE!1FkL)`?2<8%jzmxd#b=tJ^-R}Nu(K@E5J?>wWWO7*&j;l@Cc&inBfS8IZ08==*| zyO5f|zf)lO)vg1KIOC>ckTx&m-HF`DBf`FcTbnx^KQvxUv-B8=VRI>lOur4f+dDpw z@AsDs__2*P@%KA%MBBh8E6RNsqtds`TWK!D-N`5smoDsgs~GOOXs#migvI6%!EvK{ zI6wCdmw^%k4>xiLG<~#Z^;jeg0XM# z`?bXU3}>ZU+y>o@9|{pxYJ?R+?1$SL2SdJlD#R|rQHd7R@kIK_ao%BaWfVE;lZX>JS@v2ZVO6(f=8M9t>wukt3m-&eJnOmBT8FqYp( z)D6Q@juUq0c|_Wdm=rE@e>W^A+Xzn5=A(>h+xAQJCF5d7}^)-M3(Ba9?(!n(hrZqdJCrOPbS2KS; z;KzYzaTlqj;;I*|gwUE3E{In*o4M?F;^X$4d{F~FmVv?I*4LC>gIK}@x%z$m@3gnH zveWLxwB)`4?q~eX$n~2f>ej#fV{U+)9R_bZn>ms^(&k$#Wf;Ropo;`dAJ4_W+VU#} z<6-`N^exxsjCIL#(fjl{Q&Z}I zR0)cX;smFLKzq+Z>Am>poc7Zk|K)tHvDqNni--tw@6%Z2bK0Q^hD7T!oJ1MApD%(v ziSUa0NUM9l-2>}B8nM~|4y9-oX>397s-LBBUkp-V-1>c3x3W&fZ`X#9fd%j}GVc^u z*vh&dSBCA|^1ezV|I|JDANR)x!)?X*K3?9&V0s}X1*ge7tx?A<o(!y6YZ^^AcVz0DhQ#bNz>9}n8&gP8SFq_ zgq0%H#K_NWqW7py*qc*Ph5b2%9cJ7eXk|#6U?8%nyVVz()k_R338R4EMZ_vums9(> z45>l11I`b$*L^W0O*(QK38j8$78knfEOxj56_7i^faWFR3MsfbYhw^V6)Ciceb8!M zr5xQ!Nih2tbB%lBs_7PvP9m!b-Faho3SF0*2+vxU4B***rh2 zFfpY0NiUu9E@@7XbBWg91dRGVvv`azC?lVF9cg_qvlf`E+_A+P^3R;pNHW*rFP5oO zDe>0HaFX;+GN+DQDLthCcX7?{8Nv4sF-YVjkedi#)R;;)Rt5g*P=_d z8K&JJ=GPJ@CeV;VCTNHzhJv&ky`!+OmW0SQ&DMUZDa4^A)lA*HmLIgi=|i_l(m_^0 zViE45>`dSZcPwRk|jp%`D1Nm^<;L-mBsV|Ac&eb{t|XH~e;0 z-5(bZ)Xopk`+^PFJD6-JLiQF*Mxk?{&2^MkwH*fSSS?K{*moilEnOnfJQU`YydnRc z`2y0kD?gQhYwm<;A0wH%G{u>2R}dj$MAuN53TJiFDe|zIf84?&(xx_z$;->MWbQsnnR+pWx3g#o=V!K1cEWeV6QFFhY3!M zySRXeuvdUzM4V#ul*2L-uGKZhL4L38-W1<}Ski={euJWJH`2GdoyXI0MFWR_wE`L!J z!<9QeXGVNN)7hO}6nHN>;}>DMR{GC2O}r5Nt0UA&zE?j{^-Tyyk;HXXeK5*ViYS`1 z#Sww(U%0woc)CZ-6b6nKJGsptJWIXJ9^#IUY^>{irX{S}?4=Bz8&x^Ur*H}05R5v} z!7S}W$zbW74Q5Aryu;t(`p?)+oK*|BS9c@s3egd8->VR8(^esD@Axt*Nv;-yyg?np z=rT^>5ag{_OXE7}k&+O@-siIJ!i1nr5!?u3gJ(1`a?cgWKb)iPwmaUfcZ#AI9+ezK zalhY)XZBI@Aq#m`K(q49VeNy-bs6Dm%ldrhp6SUFNKXRHr{Ye9YxazFLPMK*U2ejz z5Y-{=;Wg~da;sa8cKnQmdC7j8VrDYz*Iy*%_Z#-T?l&=&T4W=kS)AT~4!_l{2=NCW zmS?KFp110;^oysUMHf!=AJ1bYBUkS2 z?WuT=Yn^0wXYk9rqY{;JhXS|72+1nI&2kl)1^z`%JC5_0txG1){WBO|Z~KtH5ML!{ z3pz4US3iq$$kLxJ9RL09&E7bSTIXfgRJDOUP(j{zJiaBYQFhM5CBdasivZz|#$>Rq zZamvBFrahncYG=KQ@_kn6TO|Jmo^^Dmb=+ALJC=hsp%%kh9}D-UP>NLj8Z)1Rl{9R za`{JpCsX~BfYgm4!1AbxrxZ6+3f^J@N~B&oIm2S0ljYrWV8$i(26kxDz@fDUc5DQq!J++qQ{(9RBMh^r!^98yT}p!Wz^QlTA9> zWtrUI1rCCA?J-fJ(al3hvV12_+5Zz&;L+ zwRVER`i<(-2{K&{EKPq5zp@0>Z+KyF(8iZt!Vo4GB^6zyii~!$zc_qtxxg^ojX)&I^ zdU1lyqe{!je^|E)|NjD&g8wCb5~^kY=V*-L{6B?pf|bXwp)%;j3FqL?Dv(6Z@xM-a zaWWd*A)!370l$@dB9&olmsu-3K{YxcGfqqdgwpCN*i6TM$D=tOds@5Rv!5LQ(2@mC zbvjmq_=k~1?1uk)|6n^CP2Q1K`xeW*gU-Gs`_n7^;J?LAH0UW-`jM_JNKYf4b7f2U zM!blV;=e#Zv^`hf(Q6-*fSTf0`tHT%SJL7HvJ%T?DxPG1LE5JcN1DO^u-!27<)etJ zk4YC-t7=Hm$@q2Em`W0i^G0D@Z)!u#PNCe3x&SQUY;ulXnT4&9MiVXQOJ5KY6Bf!z zXQQ|661vY*SD1OdT-dFR7|IGCJFq@&tE(+Lr?l^kz(=ig%HOMm z#=krj3M%K3@ay&Tm*fUxI zB(}ugS>tfEc-l*7GyT2B=$`^xG&!N1w47&0?#;jK;-zVE$oinnSE#{o_fq_D0?jV< zT!vqN6~7&U{626BB+K;#q_0k`;Lz~!hb1_B{_HdF?8mq8 zyd0|e!36Ga`$vOMk%0R}ecTfX0?c=*e;Zf#IR(`*FDv03kZsllGW%4Sig#1fZ8O7f z6JKl@2@%7Eg%x?Av)3&-1nov&TQ+oMv@lNJa4nu#etP$tBR{?Lc;4f??TxOfCGSnS zubwT@QnVdyv-()G@p?sn>CLUd*XYjR5;BNEYDH(6+gU1MXlwW|QHnKt9<7G)BMSfn zZ&#vbFpW>6SUrD@au|0ua=#`m7BvSv%w>T;eq_V{{Y!EbP|b|B`t^?)Ypi=LyA;PX zKe}y@zfx9HFyt+DH9my8^RuEy9>B~^4p zju}J~De{ckLlq)ZSk6f^d!pu-iM0pX+VXI0P+GsR&GY;G(#>OJ&;myjW&Ibj9QLX}xwh;;geG)sjb+7TJF@5;?2MJH+NgTpF2L~bK6D%J& zuL*%Oj5#fiMouECJ|ld%>UTDw?gerhOfjxXJC>DMG*CQjPaBJL6Ghi)!5d(uSWgQf z0$)95R55(nLxfmP?~8^lk5&hQf{%wyb3Ti?7rveoZ^rAS1&W5(OC;nq2gmGhCgK5i z^79;{nppK_SJuPuYD-dPt|c1<yS z{OPLyt}jL_9sR=4H|3)C7C~PZi8I_q#JBxR5j^pvi-vWRLGusG0mzTsNB|)u=By8l!q`ZOT7&E;#(rs3YmPTEyifS~m>~gB-0&S`GYS`zCSXMDh zYLS(somXrsAPMuQsk#*vNFB9!8vc=6<^dgq8$7k>jX(oc+Oll&c9Iw$RPS-60 zi(ND(P?T+etLYWfy9ZXYDX^j?7mL5Lbv=6?vhpMIhdzqm-|e|gsqc2IA9)_46oD4} z#hXEgpv#}*q>3py5+FT+rA0mee=h=Hop z-tf_y;X(K=<|`x`Q^!}-BOMNO(A$RBR)bt@cIa{pv6liEBm3wCb zjS|@0f#S*Fl>W=@FP3bS(Vb<#Nh(jgYj+@O12&4`?dsBxspM6#6m4FU5bDuc8agu& z;eG=3bSdKh^LQ=}9?FQcqBPK(+DYx_^l(T!OCn(N#KQ27_gVN`W~Wb1cG9Kf5B}-A zixg)M0=Bi z%eE^bH8)sG*PHK+aq((fL=Q=-FhGDuz!|ppj4+yNV=j#NL%_>S65N05>K!KTd${af1fXAwu&t3r@@Bl1NL$d;(pB& z!*H%7K|WKL2wUMLtnOYk395ma&wL1m)|7Gg>tG9Q`q<6*XjN@N#X+^!T=ina8 z`QYM>scq-i@4pX;#Hu{sB<(Q=09~5A&x~10~;IBhrOxD&oLqVW?@j&=Kj1Zgw z?cT>2k@}!A{0J>}=TZNBbADJZNIN*u>#k#6AFwKo!drSWH8EDQYfT2#lIZvC>gY*~+^So)WZP%bnm1ci>5@X-UkeV^~fq zTf*&v3e?^SA>hPo4O5AOFtRpXMPYv8fm3JlPyRk(IWa&Z7qyOMik($lp)rpWvsGU_ z$829O0o7qtB&}V{VQ8;mO{Iwd&pWQ*@+IcE?3pA*-Kaq`1km%$W}-k=RM%5 zdqr1;;0s-me$Enbsip-M8!@~OyIj{A6RS{(2zWnHfj8AOxCG_JrO~&h*T<4BMqBcI zkb4ogL`3qyBeQ7Q_qq-_$=!!(UTW{Nc+epo-r)6-7P}OU4O0lr- zbTZge4;p0~oS*;q!rl!;N#=kxyluIrtGPh8nhEhY^`D212&6MdU@Ru5;k8z2748~6 zIeNW{5^Fdf4z$jgQayJR>cb3M%i-_<-m*>^73M3_bq=Dla5|2Db66G+O2vO) z-fsMfKQ+>!~;6#>=VJ&02&)oO9D!$zx{y=Scf}qYQ zFlNs{cPzm=NfMJO?hvDpldBv{2UUjHY`%Lj*`0*2ZIApgp5j@CRp6D-IXx~0Z?)Mc z4cyo6pPD5S6CO0 zGnMms(yyjrWwNSul6waIxqGvDV;B|v9rUo{vj_H+iswnKq$CPtE`C13;{?-VT zDe{iM-ZOHwUAAfCsnVP9&q2GVvOy&j}6cDwY}z+5i-x_3lu=ow4Y>QJrY@ z8T>rFuRN5^Fc}tL>!GgLG^KJT&-Y%WL_bm!f~zr>8MTGr_^02m%wJ3}r-~E7+^Mrl zJH;28n+W%B#JJb>!2oF#l+^lN*^IG2mDwbGwBDMRxMz=K``2c2h;(885&v3rY_ZKR zyBg$1B#7M6R!sfj_7iLeaPShQm-*+HlG|S@jhQ!#ss2NU?mZ3?*J7$;`spJvl*FQW zu}lQcu+L`lSQ#3wtwSD6ZwtwE9@5_hchtcxSMO@}3)AH%({#qb4cPQ*W7k89yt|iD zm(z22lcBJJQ$XXTMV$}X#2>A2`L&j+C9X$J}GD` zk>R-pu%lT|BEp-XL9L?KU!BSw?etcrN@WEXv{n9e_B0_P=yR#myS~-zhG&4=!vb~; z1ed&uD3q0>GSTqMbDK0s*$#C%LhfJQBbu+pF{vW+4-f~k4p~?$+#gysSgL-`gl5`x zBg-^$C7%(NXdmNMr5G8p>F+^`0Mc+O zxk=@*27ak;jhL0aejTYXjj_3voOmI$Lw>Dday@6Y)^d67hH60WH4-x-?D2>I$+XEQ zeZajfahk0)3XQ6kdt$qF4HE5bXM(ygJD=@mbvLo2P2vz`dsuq`tGJtxiw;r$|Q>Jy?#JH#~~#!ty*{fFS< z4_a3=bcK>z9WAbq$3e!5bJ7@0Mw< z2FR~znA9W=*wimOEw~lOQ|O1MmbP02{q99ykQ-ciTgV0G5>;^Kv-%aZ!%T31Nqw-otTl zxf)D1gX%l$5-egN!GsLegN6l93XIx#hd_+Wj}-MJc}kY?L724?Ar82&UUE)Bj%ZOJ z3TtNKqe)ZndL2ItDvRnM2i@%-DG1yWHpNBbvGud{;im~P)6t@iW>Vi!%h~4}`WuGO zF?<-{6uQI&1l>Pzklx{Z{ZsDnwnB40WROQPwm~WCKh%Nka{nK&w7&XQhqk_-_<+*Mq>VaWnmkH%ZVF8Ss@C(IDf5$;*pbsgvY zs#x{xNv%2ql{Mrr3$r!Fl{h`{(VCm?oZ53-KIviwtpkdNYSQF9Xn&+dQF!nMdjC@^FP;vSmBaD3ggoIs}j{+PXYq(vPc0ybpYZY+~lSsgZ zNyQbMuFJ<6r^`k*kF_;C(~P`^$#_TLe_bB`GhZ`6eGfljq1F94(~P_;mj98&HB83; zccWM}FSGq$;eXB3tLB#fk)3&s;IE|4Z{vBg{1LUg4NSX>u^9Hd?nhNX*{jm+htt5U zZ$*wSM=zJ04}3Q%Y5pHpIpNQAfToG@u@ z0?E>dz05gv5;04U(P5}Ud`0!NBucFe0tjTlM9w3m(($2_s5Jm*(QmOgW0&{bwkwzd z!Ky1&diL^WxeVbnC89Et(W8Qn7FITA{V$W}qyK6F>l95G3V;3Z{;2C* zN#_$+6WoP{dVU~D?GSmI;G|{6c{hJ&={yb53fFbpeQD7R#@>_WvXi#ctxY};{jI3h zA2vt~q6SuTIHZJGHGptAn!WYsF&V^32~w%mW5^{(_3-G0woMEl&>!*o}AgITKgo)EI>xG0q68ehPBwbewqI9g( z_4cYahk*6rf@H)T%0itgyzBBHI)SfTyK`lpO{J$!*~w4VSDTc$hVpzqLRWh9?cfpz zw!>XIB39#uIX;erI9Sst|GW*%Mr=OgM(kD_M3b-h^QJ!zH&mcIB>+x9vA_K7=<{+& z+Zs-nb;_*)LNqS#mt5SF7Tg_Odo!=1y=K^=PII}w5gZzZSeMgUw0+r{BiPHf7ourh zoQsD9x|I8W2B#!axSCa^a7lf7B_S&V+kV67LM(5*pRJt)vaLWk2Y-7&-Id=Dp^z6; zWEUBywe%A~R4=K?Wrk zI(xpyls!|lH<$jiiA==8jwDhDiPSqMs^s9fY5rF5u9#BGFW&yqh=4`t9(RhoQqLx6 zB`K*Bs}eDKadn5UUKD6?r+dT&{5`$B+cUUs(yrd+F@X4hYhKAuB*U2-gY>sTB)*D@|(g)oi$F z%`h4WSXO%j4i1-7RY5eXqPtm3|YhEaI<1CTTe&M5R+niM>)qF%| zi^nF-JY{DCeomaNgVaD54BG7SM#=s;&+(?hd#38AMBe(t5lGf3CTUwrqyKk-u~*#L zjp9|-_Bz~daVkUY@oD7Bb47&ZI={{OymsJw20_2QcIycJIO@@rChgV;dTgz9J1dC! ze;Zqs|2ooOx#Ca^_=fGnE6P|1iDT&h1|nb^28N0uk#Md38R<2wqBd*Ff(zEnYcrvQ z>0LmhwZO;k6<$ilCbkj1-k3q?B&p36QmwWFh%?yJF}3wP3TpQ8PSumD=B;0SU?Ls5 z@o7|Qx!x98!lW4_VQL*pG1zyrLVd+=>_lsD=v5xYu6~0y#5=b zLE(QKyj5XmF!lQD@xIjf-Rxtqh?xnN$s~YFPqzETQ|qF@z9$b=@spKW2o+7>JfZ2~ zY;a?1z}5l#@*gWn!^sozNef%rUEQ%Tcqca;aJ(bXM~^QdxI;6cvtV*P)zDCO;X#AB zTCN~#Fp9eqC&v*k^bJrctVmS`nt++R0|AKcLZ@##kf`_*SY_O0Wkfk_Y^vu&gr1{q zffhcovV#WpDnD<1bMFghmcJEdc;~_VL#XGfq@~|E{@Zf|+p-YjhNq2Jkm5gsQbAr5 zj|I*+Y3-tBy6T@tl=Ol6(*g#xI*3(6#F<>D$wk=m9S|yL=n%o)&d4H{j^*bNvEYkJ zAfYhH?S(OfryhS#o>JBv87<57{#`}*tyJGs2M&f-jHc{5=QC3de*C_0Ru*`{-_mIO zI2V}L!(pHQ^SSVtaZqPBdh-8oEwX#-d`)a>v%z0V`dj^$pQ>W3Z0rBSwLNy}5-t0J zFWa_l8>_g=wr$(CZQHi7%62_v+jjN1d!HWN_q@rB_-91UFED>`{Nxond~qq!&2DTU~ofp01!8mf)DS&-YpSPR>$)Aqau(nU{X zHcb!CvlRyiV7m7$ADp76Sj?hDL8e*?SP$?t*>!&+pn|n#u^qm%{Wir~ghLJ= z8F>}w3DJ*XdmJCV%Mh;!*h;a(e0*nh#DR~UskmG8MjeOS^ zfLW4UT>MLyB)dRQT7digxxj>4L=q)n6KlqsPzCAMj6K}Rn)U}@M~Ytl5kSSZ2{tqG zZ3KPk5hsm0WgdnM=oB+U4dKcmwb_D5O`vXM($uXzjxTSu@Q9{w0f7{`vTt$3KfZu% ztBpYlo`d#;UibBeapnhHmr83jjPZ%(bDRWVgdi(+a)L>A02QjEVJEg4ZRk$d@g%E@ ziQ84j`+)>ubM2> z{(VW)zarUa%F>MCTNi8llyn94$p5q6`jPaqhWCiF7{IQ0ISLaq? zMf9cKgDn7YmCwriX#YhP?{c8RujGQ_YnBD*?YUunS$0bV>v?*#X(h)yiW9-|63Dx0 zvT)uZm?#}=nB%mSl}%?IFGezw0U*p+e(3xUTvtJ@7-Ek%TpSiUL;t+T^+Z;vYEJGz zN*}SIn^RfDFtL8?XT=@mlcQ|24?mb_%b9I1A9hPk!QVB^;6=s7GJoj^ zp1=ZkAu>kM1IhSiglQ=okiQ&i6AuK1edH}2|DrH$Gi$XL2b4y$4C<&oeXVjwlh;(} zDbpl!(w36SeHY$oCu3G$u=qDzpY4{4F@7fndRm`y8ycNz1M(I=bLhz|6Wf0~bISM4 zeO5Lp(vES{J$IE~-GW{dUnD)Tdi!Sf?H)Lzpu$**-toy6p0x%5sp$@0n7yihW;WE( z*6#KNXDw}xr5SQqU?lIL^=o0oQMZc2gT}__M|j@Ic(Rh0)9?iy`;;p?24J0nX4&7+ z%{+ue%^Q*diC(61!}Q?52wsZ2T)wBZMu>AN8QtF+YbX1>N|(?;NlA+MtQ)Gnlg_Jl zsC}5k(&O+)ADi@_KP#-X&Z^92?67fk)g5lrJHQKzBRESku$Zc8+@#QnDABP3X6BlI zpfQ8Ur2N}qj}p~psFPttoQ`_-4^ug0f~|RZg)- zSg>f7(%Kv>;XhQB%TFJwO8t~x7%NKWj}e@Jm*=NPK$t5zTX!Hgts_vIe)iLX)A*Xn zIdy`DV=C4{m|Q%;lUMa2s+J^YdD4xWj|*xg=p*_XV{}dGooE!NPY7?$Sv)}I-m(U| z7F7<1;tG$E8?6eVm@kltFQM&@Rq)3`G)(`1u5Uv^w>;#&fi2WU5%M~vtIgp08dK2b z$lr{Y8UOP3_reeO*oAJ+F;s^M5YyjyU+%5d<5}UDZUCt|6~i+_T%UKA%{HCE@vCp8 zGPn{SFLHzkxq$l&Eq|9gM^rhf`pb$U7wP)l6k{#SgI@!`Nh$EHqTU2RH@VpQ%Mf9M z<>X?C89N-)$ltIIcCf6iHMc7v)T>Py{Yjak@q#O=Cpslee0t^l~pm!PQ zBq;gxxltSp7!Ga7@O2pW)xhts0vekG0Ckm)Jn+3{6SUC8GAnFy+Znr7*Im&x%9&Vh zcJ#Qd4kz@NcKt7#_JSGDwB0*1di!GGGP^w#@>zieqgDL&7Ms^<{JB)yc3wwdct-Lz zVdS05`W^h#vFg!XeB|jJu$u!7cGEv=OrfzS_9vm#5K3#oaFiG z^BrNZE@N-jm<^VOyz*k?8PgJ+{B=e)H8A|t_* zahtVaUCnCPYRT5(k_$`OM&3iutx>&L;e|6@GuX}LbP;;~CwUFkE#HtJDuV$o8_C}h zZz}nR;tm&HKkHnUD|=5;JsL>`Q=LPL?G6IBp=p<$hjt*Lq z9<##TCubP~ik61FWa0`VsIVA#wgC0ko8$Ei=iP$9OkSVEidhEpyBdX1{st@fd0Li?zzqsWDEBh8L zw>g%YX2JEN?L3Ft0E*lXzI?c*(`cB4H7FJ^few9fFOHpg)yIE^r|(o#bZA_w?94q) z9Yirko45EbUSDd`ry3I$wnOj4Wl^>|1~Ea7K|cx9gEWv{1!Gd|uo{YL9fiF53QXHn z0Ih}G7{CKMu{ff)-iMGN^GB6FwSuyHOk;0S3Gw)&p!_R;!4^}Vli?Bm)W~nk#sAzf%K^mTJ^kQgo%m z%?kIG9W6;j{^c_HRQ^(xNAurcv^ ze3erms;NgcmGKp;Q_wEe^v?#Ey1!?>`uhk~Bap~4alPylL0!PnGdRrgX=p_yZyfBw z*+8n*R>Wx-&F>LwfjYA&Sl-H?=WdPzQ%8OxZ990yR0TPZ)kPtkMGW&}GeP74p$^lc z7G<-Brwx^!-YC&*_3fogEse=T;RrqzrUxyiy6$USBCfEZXX#7Gc}g-0J#YEuqH^J$ zqD}i4Y;#kwF=;8@_I@9lA-_mVEwnGrS7~NgTM#r>B0K%3sOJ#k0Iheg%Ia2IiEyd9 zN&oJW4Dsw>WTx6*ueZm?5w!%Ow*XXyBn;=tScx$eN;z1s!1q7xp_pK5L=nVy^K~K0 z{c-{36GtkRIQ$1^u{b$|Vz6F>KOc?II!}~vM?!TS3GB6>GjgamK?%?Txb|-VarTOa zYK?zQF^%Ysv0J6Dx^g$=XVo9J1{4|H_}eUhK>0sdlymeiulJ{ek(p!8bjN_gw^?V4*ay(`&st6MOfDjK$gz;$g@pe}t5 zqqRO#R_FkD=sbKnav)}{uNvkwpz%QA3R6blg1C3K=npWQ3Aj#=U8o)0DpT?1)~rZS z=d=$78wOSq5b0`U8`?fqTXG$OctUZXdRb#jkGRL%@e_1tw-HN59K1LbLzllCWp1me zkW5?JW*UFzc~Pi}R=7pz$uB#B*QULMP1?r>cCvWgXyI`Zi~o$avaAPJc5;W|ivq03ViyGG2|B)DxeEfSyom;T^kWNwdSUx=5@9%y; z_shrkb<6=VR{a_{iHqw38es`J>;Y3q%*-S^9*|f(n;e2EGg#e5_v$iH+SNnjKkez| zi4NZW(gXKmfh+Z2RPa^!aE;sil2jwJRAA2r;&EBZ4wlAlccj>L zX-uS;)je_30Itw6TeoZAJ8^Vyq{BIBf#KrkdR|09V9+U2vB>vPqVwd%Qd$4x-?zjAxu&J6fl^pPH{w4K zhJ4Nf+~ln=!bR7FXA!c{c_*bd$CT}*xCPb(NBe1F%k)$hCX!^7g@{66oW z>zS1i#;3Py`5KMw%={gKZ&3!nNH8a#uN61q+B(O2wIj0-mh!A(PklY^9R=jEq-=hm zDnvuEJiL-R>>f}#6B7BC(neg1*p@Jf)OJSuYVMuVtlNJ=%_ULRCTQuLyxN3iH2Uq57ITCc>Gv?DspF583eRh{t^^78bb}?2HT!b4uJRLxHy&51yuw*JFKLzy(jD zV4+#`v-$v|Y&?H?fg~=7Z@;i`^+R9{JG^fE0jop670{Z`d>qxf@fLT`BwvLmhykrbHzw+W@Zp8@reEjCz8#PZL1Zfu>7a^90{T z@gGg_Z7^%GW@%d*C$}ICaWr})lb3-ukt{B5BIsEMxrPN4F}q-#ZBZ?Qc(296hWVo+ z%{h5u`U*vU`?oG0SSfRT-I@lG;J;!(b4!(5UPd6Xip3gFZv%$E4mn-DzM$p;b>VDZ z{eOdjuUxImb+==^K#mu#DT$~vD>0>74O85ZQ;&y;VW@r+|p0!Q$Y!ZjT$2XSPr8^sDe6pfwz{~HPD z2pYmdcsT~E*eJ0T)!4Px965c$gV?i+KDrIo#ed{H;ensR;N#~J5ZGgT6v@0*?DVgm zB&XmKozu*5G@OViXD=*G^a!wtp@2K{8a!&qB^*gWUqw}C?g~%PDq;;`gMXjhUGGDa z&~S^_bMvY+C3}tQj zO84|Y+O7 zZ3%Hv>>@~99^A&1C2-TFn6YWhrN?Y@=;0n>JD0XhPi(j~=7Jcn@yv3Xzr;j$GinC@ zU0U3JEZhK8bA71w-uxrfC?OpW&N~5tWP>^Y_>FoRPf-1moj3LGt=GtfZyc6_?nRYF zh4NM=1W~db3O1&wgrafOfHI_q&~SjNOZ+j~#Lsq&~%n}DOi1Ds$CG$!tZo1)u%$dX;b1v#iq2re&gIFmNY+dRtHC~?$~~we%g!o ze;dC2Wg4^Nky1%$UUDzbXXx{Wo={}2{@7cmfwGS~K(M`}^f?KZ(Ci$h>2G9q5Z|#h z*UGf*J^jX;QZZ`~dLehOifPn4Q+jQoT=8kVW|75DiD&&0J*${CFo+eNM{P2Y)@1p& z@;HtAnWBd-T_Q0CYl!)lyA;>kMD!9a@s19G)9e{$t$QV4oT2_@*Ic~07PchmNSu6` zST#%Ayp7TltJ}x-7siZyr?=l`s36!Gj67qP>vMj$)^*0BNWo;8oRPu8$ya&`twv4d z5q!>r>NtUjOe~TdD&*ffchx0X>_%NY1^Q+c{Ar zzV0rsO9$Y)O1}hWp+Q&vIU3selQWruPGl(IQ!?w!(=XPWLH z-bPHau>}_i{S|Ny7TZpM8DjM2Ce`+ZI_x4{wW<*twC$+Af2mUz9zOGg71u{&2@)#o z2`ORGU7k8$*@JE}sHQ{JP8Sg#aS4flQfCEKvM+L12u@_+%!>lZ@!&2C?;(ti~2x z_{Tfs1u&T$;(l{o@oE7Aqe#Pd!e-y!nNP+5`x5G3=IJ+5*|$U`{gU?`&Y7{TZsMiy z|Cbu@C;KE=#Z2_s#v#sGtn2juhZ=;*Jl~tWNm%bkXfw{nAdjTgMt?c!vb3fF_OPvU z>)UnidM0PwR2A@V%U;?!kCy_m;iZQR#@IwXCV~1AA7V;5njK~B3sAID$zp_j>C64Y zB7XezUK>W;{5e{b^78IKPX{&OBU!7PB2EkS{ki3pIu6kx>EAe+OlE$obxb6a)TTkt8Zc72yA?faY#kX|WbtPv+1ajD=F| z7`IRkhBte<9Z=D$({%bC{HTvhQRnj*dG6AhM$Hst&W8KlB}tB^o`bSM5_-DC0 zKPqM2CAM~gu;hJ;K_;cRjb3~u4v-*r`f%KE(7wSVc6GW?*g(G%$) z^Mf13*`GtSSGwb)yBLzYdx_!nw;1}xlA+kE7o^WnOY$Ms->pMCiugg))U^D zWiNFfRPhwKRXDL{hrLKzQL2X+f_ejIS8p2yNfiw>YhH~a`GA&{JghkUN?dO`Y!|tp zC&hFK6A)HosdhN3?1aB~+W5}hX@E16?h$IPRAVrJhHLB&E)BO$W~`zj){%eGw=rJj z@}08=u~?=@Hz!9j)dgoaFltaR{rs`B?~V{2?&0g0P(3-k$0+4FtlZMqMINFQ7Ay*E z-G2fu9F=K63sV$on$$tr2=5Xei8lW#;?zK=g~R)4qH}m-Fys9YU-t&tZ zU5A&_QwVWW*~fev8pV+q0qXSoJCyXz0xq`?aqO9JO!RiY^0|hIl%koE@EF4keaG7Z z6xiD}1!CM!*9w2GZM!_!p_8LH4X z-V=!=BT^#HxZTdu$_kW@ye@NpqSjTCRqG+_q*jMTT_LwhR*suA-c%C>tT+=BW5q+M zKaq5FlapO+`Y$%x&fUU!;QRMTN>zs6^W{l??w5%OQcwyxgQo&+>r}6W%k?qp(i&kA zs98a$hQ(k0MO)1Xc93lwNAap~|4GiaaZKDR^?F=$9N)~nsJGracX&03HcKKH^&#ae zbStqx0OfJau##MWjYw^GxM-I2WXNILp?Hv7zH;=0Ysa z4vRN?$*%j)gvI24rGZr!XwgJmfm)N!P6Jvg+uz8@mV_+{Og|kj#+7}KXKDuDDCzuz z#qy4)|Ev6Vj<(J^Sp4+n8Yz``Jm;N5OBWa^MdVl3`9Mpz%m2^pt-pDcG$F}HHhg== z!J_+r!(Gp^j;FxWcFG+Y$8+8PuE{%||7Y*NCH^hkmKC}37o{5mYo1LJiK)PPlFp=` z6wF)ODr!(-{VUAm_nzw#;;bw9kd1j5*>>V~Olqty;!Z3q3g6S=*QKI`roCs~K~;WY zt!sf;qGQ>~(NG}VVVF9jgpvv+nRzE2ConP31_4~bbw?u<>lA^$sv32*i{?KOv z(IVFmaYx0?Mx?VBqBNL?*2xVknNQHZv^Wj~v%$X@RG0lV;RGT!+SOnqB)Gj5@%bD| zyH5FL8H)X)09&E))@a*Igd@+=a}r}`Cy2yx(WpOoYe9kkp#~cWqp+eKe|hT#Gpu5K z%!X_nsId%%({Y<=PFZ@HV3)jj_o`xFJxAYp-SJ6f`TS_NN^U$u^5bBk1aH?}Zy0#f zEB6l-R#>Vew0d_cT5~cP^nzi8U7cAMaVJ@uv5R;*|8HxcD6vJ%dGNM=RiPIb+d~2L znfv&&ED$6q#sTxh=QQ_MV}EpgPh4XUriz_*1oojnUf}2E6mYK}ALr-6^Z6Lx&-=j@ z{`+?DRzD`#?f-T2_I7p_ZolK_>-qEX*YE2Qv1qdXUSE*#dqD3OOY-RLt^U`xK^w6l zt=0a5vq4KQBbOV9UxLMS?{EqydG?7Y+vU4feR?-h;uYA z*UJ6r<6Uudp2xf3=pbP^-8hpIPq|}AGF0A3t23O#DDhB;70bP>Dvga|v6*ua=hF_% zJ{5xyOWco{*0CU~@ObnB3noKFs|M%}X;|^TzZ0xr!HR1@Z|}(LP^vAe$)Q5dEQgk( z0P+nc#3t=Mv=*l2)m$Pyob|J2&LKk7X$xMG*t%n3Z_EYrxRSY08fsdZjqr2K+Fgnv z9%lNI$TbYwp-?q>NV^za+gDEg#9FMIHX6J1GVhv7?_N<{No|KnrZ0jO^>|f_d)-*#sF%Gc}f=v>7yu^x>lGbzA1&YdZ&;&SkHWsS5&p3a+?jSd^Mn zQx)|+8~JT6q7q++B8GDrUl$=r>cG6bK=SSFA@5RJ5R^TvwmYn0ySATO_mfL}I@y+q zzh%c+%GaIxS+<|X&4>*L!tzaEu&^M-$K@?Nkw{PS&AjNVQ`u{5Yx5TOc$q-eXO&m{ zO4I9D;MB?Bu4R(;-#bfnZnbb0zGJNo4n4wb;=_2o3)O1Iy>O_O zlnL6T`1ry7{W(5=FOUB1dUN@>xCZb($XT68*{&{%D{Z=mn~*#%#|3t(I5%hu#q>!W z*W$RC5>Yb44nZWqp=c>kWnwAZxfyMLBkmhXEm2xh!(KJO`rXMS1{eFZs{RKXY%p%d z)=JS35xA^dt?9{YhdUBz#2K1%FAYyI9|cFN7B4w^Zj@V2F0Ous=!m~cM0uW%dm~6V z{rntU`NvsiuFG}`2AlYl1p+dQPo9HE7&6RW_9YjeXHW<;J_}K7!P>(hCQK(!j`(Px zs=rGa4bo#W=0Z;jnmiuFiA*4fAhje!1s=5)Wu6@}v8F&;m^Hoovxz z$%62;1YfaU9(1az(~RuP__6&AG=K+HHV4mBnlr>qYIMA0 zb*8&EO7nZ}t0%z=zxEB;`gfz$^FYj)V~|-VhNO0|Tq}jnkQ!A|0ive;Ujq}DU-uQ~ z9#Y#1FZJ|b%#MaR!(t1=&n>=`Ly~r^In-HVfGqmWpj05e8T18O-74owkf|AjXIkgb z1s&Gg)YN}*w8wKJMeJD9A!mY;N1?QGpXd52s3b3bjDMlH@L zZTBWq^MfQ8aAKZ;uaWqSaUb~}~+=3gNG$OOQsKHJgN8>UA^jT~IoMVYLo;vxx> z0UdN-h%Ck#@dps55)7+O4Mm& zER3_#mMSxf?g8H|js?Va|Hk)>_i}B8Tb`2UE5yq#SQdWrB{Ctn2474Mv@G7Y;8Z_{ z9n?tyryS9E9EVqlzLOzl0lWlyGEKb&n}nPbo@d;_{Uf3ks3{rL84Njt3gaoBb;>cH zz5mDJ`g*LuPDn z;TK)I0kT=JG4@38f7CQcV2`5kdK{ez0}?n8=RID377=~Qje-@8vG^XV9p6Fm6nh=# zMLlQ4tRBCbwJmk8^w=@16;gJ%j{cl;jCA#pvEw^ zVBp^leXD)b`wrYpbu&%X%Mf)y{Rmzg1QyC!!M2B97hL@BVVPNrmIdmDRqM{C7U4vk zvvkph?~Y6$^EI*y+-E8t;gLk22o$FMnu7>#9Z5q=avhd*3Ml-4Vl;p3>@-vY` z0vkKWFA6B;T|(=WoePpB`C&AmYKW0fFtW3Ry>S}sLNqUWx7oU-(PwMn8P<5K3+(o< zARu?R&8aqT5OvVNYeOm>_lo@O`$(yWu@oU3#a$@H=}!b4)a;$068O`e3hjxwXINOB z(LlV1yJAkQ?P%hcfzq%nt;y)@$%L}T+v9MUyf9H}ORH^?9(k9hm{r<`RR?g{J!xr2(v1gcAAe%#cAlP$91t-xKMY`_&z!U?*V=m zH^}Z2ZaBrkhnv)XYnBNd^)0cB%uy1X7WgbOV+vO+zybpx5(H~75?yqG{TYG8Z&8F? zc-YIA;W&@FV>_KP?j#A`kcN%~)>OdqHK>a;sswvQwf{p6K!fu(`7H+maC#_Q4|oKK ze!DhUT)&320x|1C96@a z1T4gtW@Id@VijhYv@8AlMJ%LQQ~{m(_Twt1=2cCoISZAOa=Y5fskxPtvp3$#sra{< zRaXP7s(^E^G8*1*{+y~S8C5s8yvoU06;t!G%>T4ktg3*_s;j4dR#gBt$wxMLd|uVn zmB4@hGW&l*4?pjN85Ps=%l|Hv(eVEF<9|+Fx}P=BQJ(Z>zA-8gwz>09zL&ySapG?H z3nvny<{pHOgVtb6h_XZ(AimiDf;7J)vRZZolBP3(24oHO@bRtwEO);4g&#Jt_k}RR zB%Hq!?YCwyN9Im$w%#enQW-{!>p+skjvmB)9BmE7sm(+YL6_lLp;*+RVH%@QYuR*T zj=n1rz$fQ`N^SL9?7VK!j2XP0K&1 zAC@+aARlF`vw#}VH4EE2IL-wds!1c86w?g+$@hZP_*C$X^R2OtOv`JM7(}LLAqKGz znT24mM0PxB_4gt~;W~qmt|!}F}(~x_SeGHMU^llSq;pc z)Ekh6#iqKWctsA4E7pM9+^#tkqw!~xvdQjl2Xd~PbeE;{EGifAf209tH1H<5b?R`` zA7g$#6?#q1v7hCK4cm!Uvxb3yCH96P9UJp~f;QuxaqYi7#8x*@i3kc}OEX1BHj-(0 zL(N2c=V1HFR9S>6M&wVsRp@qjpfbY<3hAj56?P>O68eYIO+kNXXbt?|pXU6@qGLOC zy{T~u6{TgHy--*w)XZW7R)G9mV0mzOfl39#DE9!5mtxv39!X8vlsjEBES*{I%YI#) z8V+L^v13v08bT)Y_Q@9=F1MxuH<#$1rOPFrcS24Z44@% z*22uE>v|-?J{|lPZezHvDoAD@(KHFWc7mvFxa%^PkLCJ5(DBfT?^GCBM!5!w6L8C+7nbDY0i8uBAT1 zY@yFN3m1?YKDoC*XkF*%?Lsv{r8DsQxVEM|M+eiI3aSls18bS>{V&RwarP1YARpZg z20}{*7a@Rvh^Xh8Pbn5vpP^T2h>p_ipy*A>7rC{dtej^yYvuZrgH~G6KvgzZI0liX zz&$zqU!K+sYUi`*S3l~^YCY^;78%2jd5i4r9S^+}dba|7hw@`k#e9m80;c%~SHosdwCw%nHgbiUzgFX2L*8}JpIGNnsNI|Uk(MoXbm*m+Qk&4y4Q zL3EF2=;y!mwa>|vC#Z9S|{q3t=2AF`%n83=izYs3xKB;1R%lPSaWEG(JYnlQ=W0V3w=5hAAUH5b;p$@T=K zc(@1pJOh$b>@_EWAu66j4C$xglv}vnm0uG7@cw1tR6x=o21uf3U^zf(pjWQj#EhTGZn2rg~>6Nxo& zqQ6tYH+7#>Z$>_4BHxxu&(gW6OHmaELd5FSFa>K~_B-X1UOzi7sPdO+e*isgVkO&7 z=P1_`i}j8lp4Bwk>UA+!=Q7hG zX@tPRRlK?wzppxQ=@^9MCKHZ9lLE|g87f^yEyUVv!C%PG@3!EozqKY^G-8MUDi6W> z1${-N)J$AN_!=_1uuRismI9zxx+JFL-^R+1#z$_v4ede@9mQYfqJ$$O1qEtBUnUls z1ru_jrC3)Am*61})|mEuNiG<6+t9D&oGIqwoz~WK@)L^0%Kb=< z-v!ZPyeJ_wMI~U%oFVvu0))8C4G0UUA!Z{R;OE#Q?ZGCfO=5X7`GW2DNOLE1{VmDu zsOylpcf!guX{v66Pn$16u8|ggqPqqE0FCisFHs59-3HiHBHg$nLcI%DKJz;~6&@}R zc+^)%J;@9>Pda5-hSf>z($zK6bJ+9ZIu_@dXSqcBfDT!-V1*g762mC=FSyBv3c?ai!*`CH(}phJs(dKay3+)m*V-1iZL! zMuE7~zGg4J4B%@`DgwnpJ~L({U2bSL=k#KRi=buAvkF0Qt^$lX_vQ~qqhU((UH zci@9RLtku^<~%;dIci#cLFZh*H>PI-fOp^ZPPc9|5X)B$7wNE2deR;^yY98NT6QKiiz6h-L%d#s{BsmEi%0vlx_q?IqI&1(H# zecf;nGdfpUu!(h-Pd%1SqVlBXo#n`_Wu`a)n4WHM=*6Y1%ra4P?Ilz0wVKs9UR=94 z&0VW2KX#u~Z^-sfmIzfK$(Z;?d|>|zW_Ywr8rT)+*#_&jMvkl;3{gvfzT>Ew0nSJ* z4XSnDt_819}$*x^^R z(J_Q93<-W%&R7?Jn&qHv`cw*|_RurOft|aOMyUq3wkn^ge2>^w94camPBVdSCg06R ze<}ES)}C+kmPh(PmiDeHZdzx#reHBG-l!@UQ7#!Z%4%!lY*v3;N z#wrPj;rWXk1pUMXVdn5;`$vA%_+zw|v`(})y|MrFf*M6hzKA~b&)g(z5x7P}5QUB0 z?1j{6@9C_KLBi)WOxIz#9MsH{mRdEtB8&M9q5}o6NnN{DsjSLkQy{SpYm?E?fqL*F zMU0>BWD(mCMw5q@Wv22BYL;oQLVF#8xZ*=j8sFx!B`uTy7-?eAUCMFjvW$jzAgb5i znx3o?^cmO5#m4hmyucEkl^q>7?TY_E7MyM5r=-6{mB%SY&n{F)us`^%F%wNlbzpvJ zSq0O~Raj~^H#d5(C!7VSGbOwv^K$E@ZpFz_N(9Z6*UnAiZc-K4@=F#%Vv)5roPON= z=q2xPgX%9lAg*}p1MIO!7wBgd>*{xTVLLJ;?F=ptP*_aRCSnNU}~B zpk8{0*mKL@{2-(!dE;yDlD7_I!QPBsU$SgR65d|W!>Pe~LD?7_#yXW&y6gTQ0_SwI zgII}IR40jf(D>qv(6y{eS`VTW@D`)SD{4?7{|8rX8NC0=S6Ntj&kFn%$Ogqdrh^-B zMKfz!R_IzOa-Ii^1dsVKjFWa4OJ^~Y;onp>Y>-a4Q2JR2jhULf7okEoA|1yLe+S|x z?&9%cn039S@i}L-j^B90xrWFeMrta?{kRdd?>x6}#v1!6O1m>Bv1yvE`ROQo6{NI5 z(?!ZQMt8d@R43RlxnZx%^{-&-Hf}D<+ysKClVes#h4PtpL{ndRA-GTAFE^&$PJ_>w zuHiL1(FBvZczT7Z91Pa({FZ5VIU~|mF&Jw|{hmHrjft6Yix)9ziKVuA<*+t6pncFW z%`qTeurA{JTD4{$KOf|lW^$nsQ7ln$dmm`;NEgp^2U{aIw z0@fBWB@$6QFoK1FS}0l+qsQp;D0X#0Muz%Z#U}&ZIlbohKjN1*_pLpKEd+urdyBjo zbwSseQ+DU{MKHMaTvPw-M{XKXP{-j6Z+&Dm`-#50OOQg3BQkW8jM`Nv;Ii%~v;Da$>#UO26*o2TJ< zr;n)JhM7+?T%$1NWVKRNBwqwALSkaFLNmZzdyTF+aaNaxwIQ4*bKh%D_g%hWt+_%%f*cTyO+sdXM5hMv5$F$XW)M$CF;I%OR zzH9MudnbI`HR~x*k}%^5_&4(@aL0TRa=ZjF=6bgTQRZXd=LvY%NnrIcK&-?o7XdMS zgC`e(nPgL=d^jMwaF>vI@_gQn*#)ULeW~!;=*$T##+ApW6N^V8&Xwn>1B-_$-jT

e zd^pO!-ga^pHyjE>i5t(Tin4vxQqW@;Ac~U2LhP!a5{)Qmp!{*sb#0SP0u_-TvYb!k z%yx}{SNW)QiT;{2rqJBj^Ko`@I5-wuZLA;$LHB#zQ?FZTzlTM9vjFZiq;S%Uh-{L1 zT9zkX!d5}+Vq1zoAcmMap=?4Ykc>F|Zt?>5yCH;LmCTxmkrW;G*^W3h;j?;w>P*M$ z47{>4I%CJc{@HiL)1v=f>AJ}3CyS)q&Q5BKr&DVtz=q#Nr4jTI{;1M&q9bk7@%ThK zn@!()Sleb8T(|k%OqgbBS-RGABnVHU-ahwP`3tQGXT8RgzR^y&qaiVdLtpp`u>g(rfbd*Q?p{&}p z^i(Bc8=cMmNJbuPLcHPr?KX{{g;Vop6Ud}c>W)+Qs0$y>nNV`dfWNLNYtFRPyJ-D* zTau0&gG~cX%f~CyE6QF2Y!hf`W%-Q1K97;g{|93bm~OnV)OiLZ=r+P=&~W~4Rz-l9 z3D&*F-DyTx75o^U)}E`|*Qn9ayj`c>e3cBMbM*rHqyYT%^x?bum2uoMi-e$~lPXO$ ztUWIaRqO6{UdKteu2Vu@Vq~fRrgpu_08K!$zlFSYbb23rOBmN#Y|ZrkN)bqd1w5NF zehYtGCwxltUvc`FDn3CIvnU2W0c8<;y!%G~0buXMR`x@^Z9sFv0_K8ix}7_clKsXt z*oL1gJK@v!*!AdQu#2+?jW-4K7suY2m*@z+5FZaxovbiwrb9u7s_q>=820RNaCiFo z{(eBUszg*3{2+oDVEA(wnHYvqOitSkHmc~1wx#l76x}nnf2=OT`Px8_MF>0b6~3OG ztjfn;DCh%!|5scZuSRl_UY2t%hV5v$o;@~fcVv+NDMA}_0-GuJN6E;f983JOJB=x} z`k~h7csf)PzAm6ttb?2*9?;bA4xy=2gBP0lVjj$EZ~m1oORq@aJ|&hy7TI~EV^8R{ zH25{b`uch@->!xYr7;41tba!nYJytM;>`&bt#K!mrLNumW))z~bMsa%H}G-dxxDS{ zVJhhDdi;eH7nf|bj|5I$y99R8)9)mPH*dOcr|Yi>vQ9j+q?M5%k>wmXf5|xB>?>K% zg-YeMW36_-M2lE%`?ni*HfZJeqKVRDxz^`|abXrEgp#X8^e~OctvHD*HIc7uL*k~I zCy5z1LbNJ@SLFL@S;7fXl_km|rb}{k*0W|q@}|^FJa)=+@L*Yl)I;d!6;a;YzpOc5&ivP!l3} z?W+d1=}3zea2dvqprfYy=y$6bv`Bb$ zo1T6;S8R6X?Y>v~jb@aehq#u6&(mxzuB`U5qi0+sP@JLX-K%e;2Oi=J$Xh-a6GqLF zY!tp_Dh5rh(neOtaQ2Y<8HMUCWxy_@u>S!DuL4r(NiQQ5sz9GgBP)F2%I+R(Db4KY zkAHk8gs6EFqBvHVX3i$Fy6my_J-`h711(YlWY9+zl`4`6T4#y*VL_>Hskx|HPs%H< zz~UQ$0g^klqZ`P*L@uATBji(dR*AU4*Gl)$lV&YP+oLZK&kn2S=q3URouY5INzHmZL#MTLo8AQ?JH8Qw-DubzP+ z`6EH0UQOJY082832Jo3ory>EB<8hJM&cEb?b^Y$1*55Jv+c(xR zi_e=)NYT`PHwz_i-JZ^FpmaIIJOJtWt*vFl3%-5vhcrAqDyMfNIxo8e;%_Bu?zW4& zOlig@r;zEW8ay0zdw}w@a&SHrS(s;s3+7IsInhfPc0&r`FWA?Wx_VZQJ(LcHi2zZQE~c+io}C z-DI=b{H~sx=MOmfoE$%6Knk%e8ob8El>RB}>+@iyumC3eo?;Y}fZ)R|oM4;PaE>-Z zl;AQ*gD@UB(%@Q4Sm?bBi5AM`YO85i;D0Y~w`S(qy9wWBtws2(0w~gmChF7*ycfMLD>$s-3m*5Rt zpXAnL(O)kGS`3CTI(~2NF@fv^p#x(GNe13ECp=z z>{2qC=xVo9gW!iEjmtvD+}$5|ay3Uz+$8W*z75?Ps!!e!ubBS zUDbkX*Xin+fRj3y;KsOvytdj1^Z+IIL1_oP#)jmt(SOkyZ}6LFCske82Yue1U#2$> zVTeL*5(9e11<)^Mm+@021kEO|Vn31n%!-4@paSO3BF>tyV~flU0(tS2s2@yRR1D&| z@$YDybbWk9ninDQwVHii8533snt6(DgA<5;Kevs;!$4Oj{iTbcVe=MwCia_Q=!!MG zoClA>K4^`kiazI=SHz%V-?z`%&?Bq9M=9nu3M!lZW!}NcRqj~Sv>4g}K1GcN6P2)g z>M}{G8Fg#9M0O0?<69}F$K(9i=RBd1!uCV%B-I(9B==B`9oATx)5MUWPaW{o(}(BACI9_T7x%&o#(brmWRB zWyhSCH#(N>xXQD^@T37Knybn3Fw@Cot0LRa$Qd*Ra9T7!gRv>yf(?U-c8h(b4n>6Ieu5{0uG+f(WB*?CaQKuQ#mCxLLt@{Z(r-C6jgmmn| z{jP{lwqXzOZd)8jzU`PLP5@yV6B8ePir{ys%#Rx9Lt(MsZdRSrLh&01IH8(EIg%#h z@X|{lpzi$qTLslTPX9O`@U8dN*Tg#2tF4M!Wfbh{d9BQivf`JsLGQYV6gb~=%zP0?&An+KZ#>%&6yp5f zOUK;|qqx#tft&vh@F3eflK(gU`A|+vF7-3e-_N*`Z28__0W_${0p$Tro?ocyA87nSH|_BSgv|8)D^5JRL%f z1-7blmaGm5$<;_WHsH_w#@^#d#5$+`t6!?oQT%`Jve{52MP z@bD_L%NIGbcYI@YV}%s5Ja4kHbIkUk>G(q%?>oScHT}9-r9-@6i*OV^wkvW9yR#|6 z>p8C-#^R8e9o&Ym_IT5`|XM%+i z-U=xnona+%%dp#?K`npsvXDr92NRMOtpbQaH?fT8sr`eut{m$;49Cx)_Wpt*;`3CI zJbdA=#@2}FGpO-(;|2~|(fO|^XARr5wrv+WYku!MXm8yy6l0WbCfX<5>jF?uay!Af zr@o*Zs^^K}XDYYMlQy`z9{rhJ@wBn4?cn5uYxQfQv=^}lYKa62_K-vz?vbSA_o>*F z|M84?xZa_63a@wIOHYi?EJ8{~Ax8bOoA zo2MWi*)Qxz`trc@#Gw%hyQ?OHY($ zt~ycCCsOekdOu~;*|!~$B`^%9rA$AGn&iIM^iE?xCD&;7M))sQ0K7;69WrKo0O@m5>irL2BcBE)Q<1L$*b|- z#Cu1Em2J1$D%-t4N^*ERdpwiws<)V-r|xF|gLlKkP|y&qv2q#6TZo}#Da9i}JZWyB zaq_ymzl=7KwN|S@%e=B_imF3BNHxXDk5nNsEtWRH5lGVGXQ2~+K9l$$R517demubP zu8_eDQ#h!yWg0gI{`N7j8w&MKlU2njbAHBl7NbM*F_2WW>^taX@JHe5lB8BiXN}xR zIVuLX^nG2zI*6!s%6fetlhRg4$roK*U{h>N|8YX#=*ZFD@j_k;Gb=iWSi-Ss>g8@s zxvB;t>3MbeOvm^*a`jMH83#>>Toc=xGCZPqJN{fQV~?-l_n`{iy9sE+ z&F%LEZ9iet)b_ziZy-tXh&-*Uh_T=29%^!wc#1qc4#C1Pr`EK&tPO{qS4mFp_^WI$ zL^fkuA*syb?Av^Dxpw{NQE+jS`y#8?+NQz%>!Gjr}tH zs~Ao@@X)o;XFt7_pcvT7R}%k{kZj7zu{>A?ZJJ6?Kbk3fV#?ep1t*?9cQmr;dcxAp z^moVWLl~%2cs>Z43Hib!>G$-_yivM5D^l_h%g=JD-cRbp5LhhYisK(c6f;YwAYYqN z&2dnMyim>fci`VXHL1&K7W=GT`~_6Vqyy$(AvkO9zf*xwaTDwtk903W_HI&!2U}2V zhn8D0(ZVf7Mh`t$?3S3Ois{32Nzj-NckY6-xOr)Xan7%CjIlpnpN$`*H?eVXv3{?| zSF33!wf={rfEPjLxM#n2YH3Oq&b1)ly4T-ER zoV%n8n*-Byj>}3NY)bt~1x5W+$NTfQD7)oO`v8e3 zunue`+}6ns@Q_4JY1Y25>Kd8@JuMYMai7$P4{y#?XCtl6#Pnk0wp9^wc90({XxJ4& z$qo29i1;yBBW|K*TOQu+YR^on1(Ao}oN$1k8LpGuNm-+am~m**<@f+y3GqAsJ=fWy z+U(gFs>YzfElSC~{qYBma2a{)gn+bi5MxTo1(L+l$gPF@b+=C_Dl5S{Q`b2-pI3L| z@a=O_a4L5o5!P*kyc^M*HZ+|N>CjX>1+K!k_UGj70@>VNzqgamS9BZL2E!`>s~o_S z=>AV7{v%PQzxi_W-HZu)_z=cGXM~ByAoz2&7(v_+^tpn9xVnEBX2^maT1}8rrzkxx z_5)NNF%~OJgUgWZ;$?BI)wT;;S*n<8DW-po$#(Qsdy={AlkJ+Vu>UnTkmS^0C>ies zTxg}h`t%Ld-7oI_It7OAI`5PPm$hH~m@0KiF#xyKh&Ev@daF1_%*59lxd!)X!L>l#s4XjQ zw69EDqpYqMqLHu1t;(Lu`PaOg#lUNW)(ibd0Mb|i6%-nwP(sq*- zhe`)xBnQs3}0YCILnwS5t?}#S0 zLB|L&gmFezYTs<9R~Wm}qA_!o%=C74nC5_{tVUZ9o$vCYKmxR_ujE(VP)pYG1neLk z^J3W`U-dP6?^v8T{%T?D6PvJw*6(^2;p(**w|1S^Au=W1Q5w6(HVJd!B%9C_tEV4z zjo*6H7(3pG8(@;ithHR459uCuP~-fZNw)gTi$9y0ye?H{uo_<_VKO8oVD(v^4_ZSq z)1V~K+o7=eJunVAGE>^N3^mRFga|@BDO*-y5`(a?I+5Q1u!I&1uu;UOpJ_&OkS>9v zM%TZMzF3D)rY<~@Ce_@!ZQ_*6*iKwl=%UhC7qp8O;QUj|N7jHi-S~E1TJshH*We&b z4+LyGbV2q?#xNx*PA@PTBRUFD##8X-blj-=~q&3OW z7b*R~Q&`mXPTbrer)SYiD1p0|u zzK>!2J(rJ&uYzy06X+|lZ(N=PkHt>C^Oieyy1yH9w9T@9rV`iZL+|)>@3Y2fk|Q@k z{!*t(CJf+o7waTRTjVU4E*5K}O?PhXHk_mI3-TUAjs&ADil7aqx@rEP+FOV@AHQw# z1j4k^;dqE;Yw!s)mz3>-SAbK!a(J2DX5~z(OpjXwsJ3d3sh9q=aoj;4jiVCH869iON zQ4ACrCAv-mgI;NCI2nuQE*aA6clOqF$bfzf>pgCk5c#H8W9G9% zL?u}!j_WuPR!Qr#mUm!1cg z74b#tixAQ?lvih4Bw%&byw>U<^X<73KAqzpe@9ui)8g!(~(S@dfDDKaoW`bQ`qs@cKPy7zu)TbXC=rJ#E&Qdb&`0>%ofDjjEia zQ^G(~oMA|M&xuG&lI9!K1q9wr$9(H$x>$!W!ygzeei83MO+_hBLo;=?k+kX*-M1iv zFd*a-i2QCSJs`KKbI!Oc8tqt4?c))sB<8pJsL|OUxl$kgC|S>Ny2{b4E2f+NNsoza z!}3i~!wk97DRCzJhrFW6qv9;tXPGV5mi6AkJ`rSBp|c0O`czo0>xA%&pRKKg(^rO; z%y5!i2IMvb!#cGw(dxyfI5|Mkf)2*JB#T}wslh&z3UXwDq1 zAoGw-A6+L2y9~9 zf6WLD9dm6!rdYOsr3+8zfP(&&WTrK}F{?$VoySG!ljp^XM>H6XN;`2GhFP|weA+T< zI1)Gn6#eq4bZzb7_ZG`ZBwegE29n53fZ}LdYCRjVw~CLa#ol)Hs726?Cz?fOV^cu! zRlxS$VIQR+4a|S-<7E6_#oQt zuB?snr?JWrA0T?Bdgo)=G5A4#tXS>hM(Om&s(rY*K?r&mMO`MmW*FJGm0fQShDJD& z1&e&(3Jg>ry02FtmaP`4)!rq9eH4S)M`hXAio3%Ow3oukoA9sB&r^cVnf9sIGTJPHLLD19{ z_jk6O7RFPPTG3b1u{#G=ls$FzdvmB629)eg$%zr?)}$V)rLH=UArE~DvQkkj?2QEl zoj*my#I_sGWO>Hyf9K!BW4HSnb@B==EfZRJfpf0?n&nqN+>OaR0RNcjd`TM=b!V3; z2Lg-uMK9Bg455J!UZi>MyMU-A9#azYl2AV}&=OesfkJ4z&bXkZMNzZq96X|pn{j(H!e)5z{#6oU4B~L zbQ_}(;Nm*xku)xQ1MSKSEd3?s6A9*dkZcd-o##jbV>Bg6r#SmA{qLXNkfH%8| zwwv1kX^&0FpfLiGu)4@hUi>Nmi$({*RE88^^lRERZeP^8u=9=p+v9mB4qA0F0}%l78+*pGb%~32=H)e^(=c&8zlTse7A

e zqzsAJN$rkaR*N7+k4cmk^K16p(mQFGiFu2LjHg^IzQ$POg==#Ai~W2!B15$ove9$`4vJzo%w2Y8;+}^MuZMoKjLsBelYQ`( zwK>Xah#5pZUl_(kU| z$*R2F9mb0NAs0k$^3X~z-ErBhhR2WYuF?kCzck!)sRNr|GJa`{C-?ER`2PjFNbc+C zg;27YXd7;Svr8?8v z;F$HJRn$o8nF0#k{p%f_cu;zrR!dU>W-TFpSX<;!Tfjq`Ia&Y+PRbm@x!$<;o|jIk zWE#Tvf-w3gP06S@ZZo84G*Cj!=+CbjYqRvfO7nCZC@5MdhqCFhW!GrSi=Z~Kj^Uck zb!oS;YsNY%3>Z1viaGOQC+%P$S?duRFRtSfkj-rAilbD^QAGA{~tr$V&SAmwVujX7|{OHi3k^ifSsL|-M!xnh<1?Y`FP z>Izcn6{kG;OgZeg#CXGk>Y6)YGM%wXegEUBA0cBnX|0DW)c&6J$F+NWieI1>$!`YE zB<(p`aAI-20Mr}F=tRycGIdvquI4`m0-ux7_n(PJ|CID=ydF?TReS5b_hC8mb3Vgl zvTkx#>qe3qlifvGU8m`Lu4oZj^R8{+Lcv(BHGFJ^gq#Dn+1f`=i<8sx=xlG@svqVs zxVh>KpScT-Btu-URq<~8JTwj(f!`jQs59Rf$z*YK5ypeqI}P|=uXXJ3bT|=c&KN%t z-Fw`GRB>X%$6pOUCz^jyYJT(#YS!_Juk}7cO9R9q+z>P7rfky#AsvD`Vt#Xa#q#g4 zyTO_;-sNJC>jbm?6{Vr2I)p_xV^wk1Sn*2I2`cj%eV}RGl!yD60g>}&IiK0;kcEo( zX$Nt_Cfsv?3WRU>#O0xL<%9e)$t|U-$_+CF^v33esBRvl4127`JvP4>gUxv}uVJ<{ z3CsyrTcj8%r!?`2hNXje~w~}`tyIUeI2-uj!H-f`aC>eOg^j%-XG-p zqp14D{Jg)NLO;KcN|KC@`fm+0J1fYroruhua1RDQfDL4gSc4UTYDIK>ZB8Qheha-p z(>wIDxvr-Wt@Gg zg9O@xj4Z-f8I}7l>}XYo<143LWsErNi7=W<)7RaI7n-@t?pJWU%o}fKJ!$q~c_^6? z&)rz1HmO*uq)g+A2{RTHkfbL&-S`~aft4VaSBMS?)8ff>D`QacH*Qx|hGu>DkikCq zgoW@-;aN})-&>Pt11C1u-^(Ymj>sz2lj)&KwRJGRZ3=k!!8>lC&4!h_-=lC?Kz66U zjZkAy{?3Y1>)U=|Ft&geXpqgE*{GkebTd^j%I9`8a6pAKhZH3;k-$2Fo|Ch74aDF$ z(I|DYEOdZ?uwq@?x+oIy38Asj80N%+^zOzUi;c*KrzV;xSE1-WspJJuWJzWH-3k*B z)3M-r1gqDpquo%n=gMIWS<-L2!apthn4>D@Oe`%@Hi$+q`gKYxX$@0@INr{p{1Lx( zH^5xz#e*T<#K4lF)3{V%9TZhF%E01VwsXDtEP#cu5!^N_tL;FHBp+drjSfjm2UjyRveZF)hNG+BT^zf z)(A!t_c-(2$)VBM48XunJ1set%&1|1jM3lrx-R?FKT%6*A=faD6J!ov zUhUXys%vvwH{9iF-DPV7IIUM+Iie^k;keq#S_Ww)YnzCm^MHq6zJ+LPVDe)s6;>;d zTN-Jo0BwjLungYVKsxX@2oHAUA5v9_t)Z-dLeq3RMrA|F90);j9!;N#*)3#4d+$89 zLr&T39#VAH*WuoLf=?;?g6$p^+)K= z%Uu!={SCs*<~A=i+p7FR77=olmR|;u*j})aLJD?)&6YhjPOClPb-vTytU$uoajEn)zZ=J*-ag zFwzR7^(W|z$z_w}_+m9*n{yLEpM4Orxh;KA!wWEn5eE~@~ zt1E0yG#p$JwOuj$mTstQ?aL(7PZzeW9kthP^;H}wCjkngC*axzJ9ST`1m~!17tddM z1FGvJ{I$+R`y<;TLaujkdL)qQIljDhl(tW0j?Qn5r`?OjH!r!bQ2_y0vBw{6flfXkF-(Vf|~xEf5nEJV-Nkk6jWu1*xfJJC}|JGI!hQpZv=|@oXE(M z#VV6W(5hZ=jKG57yB1AsEXe9Vi95W_22rfGqWC3^b>*!?jpgRi9l;Ci-zrKJ8+C?7 z*^bZRF>v1>n@NyF&y#17owMv~4^lQQu#^HQ4Hyq(NW`MEf-X`E$+z(ZPk81EpQ-7a(*lBi9 zT|+XtUd+Z;G+F?EwUx&i^9-?{bZ6T77aSB!PSt-pl&qO?^e1 zfBZN{w5qtEC1D!xl0U8t>m136j$l|4b{&kLJ{+vKvpBO|sEvX_Z3gbUzxmx&3R)@d zbM!oT*ROfr>v?HI_;j9M99yW2O`^LNIpxVVb0;YTgJ+>m{_~2J5(_~!`q@E>CJ8RU z7n~XU+xXRCg6$~7>bh(4TAL&B@V1*{1xDV1r@QlNXW{$+Y0rYt=qsP_`BK7lh4BxM zaR+|5xy*f_@PGir%SC4pT;Zg@nCA+i>W!eCHk&;17>qi7?mJbl)^w5f+K|R|o(ad2 zLTx!y;u~Tb|8V3+TuflKjThoR5hJ$d!}5B9jD;{-4KrJjtTBd*EFiB#Ip!6n0h(*? z<+P|;-DgLcMEq_l#Jz_A3DI7#K5&0aUu>uM@FDutR`4bAX3KB=^9WJg?tln#n z-q*HMUu^XDQpib-?7xi73|eyo#PcU0G(3cj;3ZNP5m(m^rVP$q!?~F8x}R2OE7YeD z2$m2;f0K?RTGN1NbUfAciZY)m`6b&$E!^bQK}ZZ&6`-G3!eI5!b(LL6raWDyWF&a` z%Z7pr3IR?;IGR+V4e1F2@!`-S0@2wYqGKDu0}gBf&_CH%l)nwlA(R?|N6>^e*Zi&> z{YQ!KZ3OFK_zR(W{~a#|16dZcuKhBPk0CQ9ggJYugry{&6qqjc1Js+2o%>7H(5uBxatRZSmA9f(W%s+6- z+hPdpQLZdLJi}`^y&y*mNX0JP=n;xRw3Nx_K}wVqiV)hM*!&ZL1u@s(SVq5obfAqM|+cinZ#x}$w-F7YMeZ4Xw2iiU=9q-6XCC)5VQ=xonVxn{+( z?d2}6uhnGeP>VQN<{F$|)r>Smm40Kj6iX?k{8MrCEipQI45%MD#%*8XnYxbVg|0)0 z&pTP*Q`m4|Ss9);FdyQ|HJwEQ(qLE*9u#y&oVA99VXy9a21fJjsUt;gR972{3^0M! zP?el$W2}*=E^ik}L7-`lNdx(_px~D~%snWk+jJRRw{E9Va{gFrWwO!ruob#zEU0b2 z_?XG4rPkXR)Sn&8lcgZ6#&WaGQdNq)B-@E)C_BZbT6o{N5TdE}oRQ`g6j4+YdJEhDMxjy5VcPf9jm%;O#YctHZ3HcHUIW zj_SEhhp}_%!avZyAKvPf=QHHIVa%qlQz0@>GLt!*L}G7utKwkc5ar2T`;*)dv$GNM zyiKmVX7(j>efVKbhRnTq`HM@#%@@Wdx(rHRAgC3-oJayUx3LQ5U9=|eCq#Aley&Y* z#~t7qEm8P#i6Xeem9yP$GLpmSC&q3AUH`Yi3|#r0Mc4GaL)4gSDtXb@f+q%$CW*Vf6A z{xk1ZG7!saH{1+AE@P6T9&|B{7SBKRL?ZJyoH_DxaL!D!7 zq(jyQGR4YTJL+r8*0$ELncCKsN2#XL8FPkPvoSfUEHw!L|6TVs?Bgr^j-S3g@*%`C zdbDjHKHzg8cw|t)yewl$oMYcWP&;!|kcXc$u}-G^ahbz47^mSxVkcoEqh>X*6~bxL z7E0w1J_MBiUe50myxM7 zrqE3b#;K6Nccv0q;;+j5%KJ>Uv+-4;ETD>xfJRVjtH#ULTEhgHU!XFWu#gSoqX1Eyb%#bRk6vET86j509`;qQ%@3c z0?lSk4TrXuV`(y=?keYb zIFaog4Pf190Kx~x?!bu1zc&pe;&Cb{imL29oILChmU|gTcU(JPU?M*1v63o$e>|!Uk%t&VSb%kV2uci1PD|m{2>h+hA4_gH;8(F5)+L7GPoTE z>nqCRvf7xi_Fr`qNgu&p7T8ti!ci;8D$Ves7{fZEM5@MfO#7e} zR-XY%JPY4_<<|rlY-gHDIqYYe^}aD7-d^}+rKn5_g$e5i#5wzD-UYka>wR*lD&TYS z1bK3(f5Hs1c|bEsES`mU_-SDFdYg2Kf5o3A^rioX4YwqN`AgRu4rO4qn_~1h#dTZE z(WIQb^h^uRI4mPvA0YE0#}f^ehJ#GUimP(m2$k%~ls;t~K;zpIAYJHiV-%QN!vDRo ze((V%gA18_NEB}bkx3O#hWt$xWQI*tPWVn00!?8k=rT9y+UF82$HKXTeAF0>gN-1x8`O-zwO7^aZGqJXLbIhYX_UY8B5 zh3a>M#U|S?rp1_lw74;j%$Oxqm<45!yy*P9dL9M3WBC?&uHkEeksIw0`SX4_zg6RLcAqPkTo>EWLu$Jn6?~{VS1)quCg4M)~IGsvB5_O zNjq~`G0{f|N_0@4k-i5AiCORcRIhwbkx{|`?~(p{h&0`Y%ZGo5o_t(6-q2go9T5X4 zd@lhzu$7vHdG7di^5HT35N_s6MKN8q2YGHJ9+Sa_ZqKPlu+UrrvyD`mM%Q&%+Dbb? zIP-nCKeO~Pi4+|L8RSu(iFXX2W|}BN=5j9+r2;QD(xha4PP{~jhcEdl?Q!cHTGKbo z^5dNjGO0{{gG(M3{D*jI(vW*-3?t9sRSMKR6L17F9qlF7I=}yMI~+^k{RH0iI+EFP z#;7DS+ZKZRVh6KhSegx0$47b!ZiWSKV@OaTV_;RyV_28IDsVs&4)DGqq8&?4k8@6q zJcQOL!QSah|4mq6A>UatpW(*g5Lx6kUyfnn!;R^d+W1nNicF+`BiJu3!!kKW2W1)D zm>#JM8Qoz@;C>70s!C4PLqXNzq8+|IQLgfaeNdZJAw&ieusG^Q>G)foWEwQjSFiL_yr zh>7H-BosSYiv^>p4?J?Re9?^2gYDck^@R}Z6Bj@`Db`7mg?z*9YYluvR7T7J&uyfZ zlK}}+{-pF?9-?08g5t`+z`?NDo}aCynw>I5><(uldG%#mfB4nIDRLU z#$p?k1z#L?XE6nGS4MOJ0flt}9m}YvR9HIIj#$eN8aw;_WzD9lV3pOq&*TOq*fJL( zE+i&a$2>yyI3=y>E$+eGn4z-=zr%4nyiKm&Z0MiM&e&8oCa=8AnDbjwnS~x2ibOz{ z;JfUaG!41;JHRChRmY;ven4b2v?D-s( zX%gGI-F4#j*^eWF?2pj&EI2kA?5B5sw~J+gbn>MHSm$1k+ngYs-F(*j@|a7xfjc9U z%|-c*5M#UcLPsb{37H1AWNesBhMka<493RKeO^dZDxlp|oKdxDgCv7a!JFlQ^_*!d zNnZ1>3#0iGY`vDQ({6nZ@?dh%VY903B1FVG7t#hLGGdMv^|{0YKV)j%3VaIG!?usL z5{}f4uEC(G`LQMl(AZSekyO2x<^RHB@)Ok0bIVpotPT1D1);|HONBba0XH@@xY;QR znZo`8I2+0S)DUish6&6bSn2JTbTyv^Aj>2nyXsU+(^NzwMWpeTexg1ulo2gCf|eBq zDF|-#EkZ+PLnX_Lxkx2hFSArayK+kmuebMlTV917qd3EK35Rh*NjXZ`Z=^!Y%60LSm}$aeCF;|FmZG8EBh|t(VD^zth8;R~izqAUyfCFv z*18dss8vHw(YRo78LZg@yDW>Jbe*77JS)>~kj4kqYh+zk zU26?QeAsCXIOqtU;3MTIn3++O1>Pv9axXK6w;P(3^oQGvi_v8! z9ir=4;74MZNb4ZsycD8 z;yFuf2?G<;G-860#1=6+)zyo3Sf>2(H{9!!K#QYNI*WuI9$o> z4thnSh{M3bI$gV9f(L7$fc)U{tx-Ve{;w{)BtE0FoogsIdukj=bQ%wN21uld^dY8uT=fymwUl$-APX9%{U&j&wQ7LAkZ=vt zzFXDjt7M=qWl)`b~ckv`>)W34RaF3u!7*-?`>NWndi*Ess%)$RPNw%!~%+ zo0GH}O!Z~4*9VrsHmUU|8KbWo|3uQnO^fATUN614oW)IKhWMuP4c=zm{r-tShGvWQ z4N7weL|hRszP%q};I5e|5b$+A=lA}W(DbmzYQJi;c-XE*MpDH;FVnEihcpFgsPV0z zxli!`i3Q}gj!u-T?EXXG5G8)@b| z7-Xf$`}vnq_rnqX-=L4%WFuQXd8iPuU|)K@4tLPkiyC))wm!#<637z^NzblO5uI9- zesR2gF=w*~Pa09weXgA8Y3JYUn}$_&_zTrIO!ihA`UocKqm7?7E%s(9Y+V!*pXgiy zrf@@m|C!6v*XZQX9xts;09)izVh|{>Zfu-ZW(X#uDZ+8AL^G}!C z-9lU8i57fR<7`X&{#Ps~k@U5yH`jAUIe98hSDxwk=h-%XZ*o)KV$K;!H+G$Sr~Mpr zIyxWK&6*Zi=@M`5Mr5!j^*C4DMY2qssyQh4J$NZ7E!`5t8d`diLrEu+eA&=wr8)MK z+uds)i`V*Vc<+lh7p!=bMoQCl@>!Opy=U!E$wr|d=4!r~jn&)QJQrK8pcAzoG?a+^ z#pb#52kr!fXpBO*PWv8+V^ryuh3Ofd7X0v?IKKSM!-y@{Oree`1S&oz*Df2E68r}} zAP9sH+*+U8O7gHQm@DtkeDf6w!9OC#FCPyD6IVVEc*~L!bN3s^m6O;BN9sF7UOZZ} zmn)YYPgW({cT5)2opAR(I<>E{Cjh2zk$=1aCnjohVT`DQl4miQj75o|iDC?c71DrP zK!X-BR)kGu7ukrr+Q1*sQUtWL0i->x>3K9J4g@{6$+;zstO>-U?wo0C@8fjb{J6ga zkHqIe;t&8^K%~F+eDd^j^QuC&|7L~T4$l2YGHVL0`~>8iXPI3EB!J`l*4_eYsw3OQrkt8SHXQ5%kd2JOOT1Z+Y9o3QMcO3e~tQhzB|fm?^w z2KokEcXcTX(!h-5R#m%MbTGUptai!6k+DtjxOofIidc`y7FHFDySH}@X4ky(jkbT# zV)mE!lxZ`fd{(qx{p)nd{wZztbC4@A>N1*i9sm|la>OSHm>Y$kz~|2D;8o6&{Eev% zg}jq)!ZLM)f@-piMA-Ohi%(h&3DA;t`v6p5=u5Xb?4sktpm3Vt&0L2`?K!5oZ3y)( zY%6sDQ}T4MeF+IPrk)#A)@eCs>#efR_+v>v$xE)ceKEh2(NDd@bIam(QzGcf?|zF= z9K&GcGsIY8Rl*wkHDB}?9=~AZRzO>kjVcomuS?HiOQbv!UYw@?I$hv7nfAa%J_h=z zdt}r8UjezgObzGjj^0FTL(_Ns%a;hJ8p6{C<&l;;WAnM5JCPfGsI|XI@0-mU8#%Bt zfMG%_u+_&dWgK$aMnBVU+okcLDzp79w;3{@IdF~ua%4sw#W@HQ(ihR+Nr*-VX*<(IE`vU!B09JuA!@X43~nEasYwbuuotO z2aIs9CPH<9N`m>x>`bn}DlW`>NSTp20cBzDmy7)OJiJovA@v*d=S#-h4b~yyAz_G4 z0`2~f`cK$r@YEwkX7N9_hcnHjcnTFF$fb~tU9LLKPqO%|KTw;9UgNXhalTlrixKEh z1B+-H1Jv!%lY61t1*-30iCK>(3O(ukS2r}&VrWwYX#l15lpH?(Y|hl0wAU&L zvKLmxeO_5%)z=D!_21)1^iJIwt9pw>TIK> z^>8yw+KfI4p#uU$tbJ~o-d+Z5JVE5xYws>R19FUO%B_AcqB#ri_7kttPd$8a__`cd zg>FJ9yHH{9k4^EWKR2@4V~Mb`&;Ia4j;?m`P2J|;mEeyT-SrB4R*5glUw%GGb+7*o z_o(|*utRqg9)@|%7x%IWOBI=hffCP30_1lXHb5Q9k*h4MCt&4YbONtNZzMqbh{#!h zwq|c1k_>y&!sCPDB z$X{YlZk>DDR4@0&&Y?4zU`Ul=@M!NKEMY0gX6dEk$es6FCuH*dQN@an$k1&iwt zjBx%E#J2;@b*h??tWG^bh&Y6ceaplb#kcIwNy2$+L~rnjRjv)VUyn_Bk|(vuxR2^+ z#cm90^Qn{P+rjE+v^B}dxVe?^KRt=qopgv^G3V?v=E7CX6jpz)t+NFU)H42G02n~$ zzreh8pJo}mW0|`Mwx5NEh6?bO6Igg6Q6f)*b;HQ+HXF=4)qARcK`xqW+1c_+ZcwiH4RPn;bT$spE~# z8hF}ojc8sFyActn-w*N!u`inUUmGk{vk#v4o1!q9?4#nj#ihSQM&#bq&s!tWEd%}w zr-g>tIjlj2JCs5ZGu!3X$Y5N#00o{hQSyXlzLZ**HXC4|w)4IgPV5rr0nrdv`R|CRmV>Xv%m*r;P`@CbHiZiC~s%=fq1+iw-+|z!qho<}3Ix3Y+iQ{8u zy)vnkt@ubS4mI?`UdSg_{3RvJGLxBU)pAfjOIhpR93VF>bN%Ph0ni6#9GEu$6)NBX zKMR#zU-3IAIv!)-Lu2TgWjQHWS_Rs3z$Ag&U-;Gl$Tx6BACDRbd4#P$`Y(BdY4at} zH!SnEB{PGbv&=u6Jv}9X`jHKYW@St@Z1b1Gl3e_#hvJrI)*RURw=&3%X1k=lr4GlqCK+dl^ z)7Y37@UV06un)N`Orlyr6Q<_my2?!j-C!D?mUQ2*%QAmYWI^7qiadGWEfDla%Fm#6 zSI?g`#BOaMjN6Ia*wCz){ivVD04_~d-41xL;Iq8}Xg6zylub2vPovEH)sH;iV{i0p z`)mLcX)d|w$@OezdYVTuP{?@X5fLD>=GrAs0>Ht{B*fTCcd~1;eF8b z7cst!&hcBX{Eg|fTaNC-D(_fv$Jzb)y)3lEJJalNhTKh^LtBgUU&eF=(c|Wum)|@m z7inq~wo0*Om7!{+=7i?_J1lT=MsE3V`*J-hp8nxMYZ=KLrIC@leGU1z(0c-wW-6r_ z)RqO|gRSZa;+>0ah-)!2%-rkoQ0Xs+b}0%(>*7!>DOhju6C5pBLBB9SLOvZLVQwWZ z(~J7og2^1^l?Y@3i3ZCA8A!LRx1vNnCbIL-O?J#(K$EKQ4KFHK+3ZIL$5f3p;tixZ7dCoKsr5}SVt zHo(cc)-@~?56(S~&kgdpN8)dL@U}>5JIvEQWqvl6mql=@$bF0BP~*81@Sb68h~Z3e zTxpmig}G4_C)$S#q4OtPg1@o9-#I=!Pj1HNnv zV?z@~o6%dp)VT%TWbUc5;Z$iyNeFD6v{BWc$1Ea z`%qw`8(>|b{yb;G)Fwru=|9#1)(8#9x!d03S~xhX+qGLXNimQf(~rBi_!>KsYl3L- zUUh74TuFUz_Fmx=N;dizeF30)wdtIs`AkoAKdl$FPsMxqoR+mP6Oms~;b1uOew z{J9`c=TD#Pz0DyRtR+x#h03X|hs%f+$SjWUJ&S_T^c#`IB|W_{6?gSZGaDOU78N_g zpY97@vWT+jCrg?srsbbXIEHZAtb-4agqIZ(a#Xe0`(Bz02uN68ONWBT{qkw!>Nc*Z zzqF$xa)M7DD^f^Vp6-2oW|^-(p)I@S19{Avd)MjhMPSW;G{qSd-|&nU*)|l1 ze~l}FXm%#POwy!P$B169C#|*P$uhrYb9%Wn02NKpNxSzV~z>nBH=s&cA9p(%%gQFTr%p)yOmli8RfE z$@=V4*ML`GCl6a2y<0moU73`^L~<&&9`iJpJ(rzvr-Q}z@71e}s*)#v%w%$_cOx>b zuItUlN(H(2{+hhx>ehAm?^o9J@>{)=DAB5B#vp|Sga+HW+0_POA9AU}3A1)@##%NM z;>Eeaq+il|#c;U`yOn4h`L>;jY<(Efi zHmmwsgCC4a+7B~;iZ;T#F7;SV#|V=7lJVP(icx21u25xM>Cfeo>NKgwcO0^ zG)IKqkgn*VceoLnH|%&+E~F zpqEP`vBcz=sfLM_2e@(%1wk&h_UoDgdq@Zc7R{f|Q+9W*Qu_1^rhu3-;J|6DlPYrb z^k`15t$cLmuiHHrgt(N2n+POnKRF^NvVa*SRFI5O(^AwBnk|-qJq@NwiTGpri>FS-*ocQEoLw}Tly!zv{nbzq~v}r-gGi1h|xjhZa-2po` z{jn;TmP?`p9rtuwxY56pm$alP;{XK;%-x9=`TE#ljEmv#;MpK`e~eVAgHP(IA`M+&O7oceW3My*-7 zNrVf|p6Z=d)k0Dncy63lC1OrQ&w^%<7>E%hCeho_R^_przN;Xwi@;p$1 zgjkf<2nO|2E%s=|6Q~q<$*^cj`rP)*7{ow8*G^z>unU`^uC6JYM)F~ZCupV=?lt2i z!qiN~%G0@(4zY;?Y{S9SXtc=VB%U1`C^^x8I|tk9=^SY0pxm=&#g4#aU+fi%4wPSp zd9kIq!3Vy5cX<%64h7BZ@@oBbC$EM-T(jbiCu~4rFgB{kW7qp5Q~J3glMR0z#nJ0m zEI)Mp^lcQ5-%&2OXv6|z@FN$776EL1#-UPKkt6~r&A9d1H|+9|Ai~ciaIqq%G<<>U z4eo!$&T-#MoUnYojH`pLAN~5eX)%4l@-_MT+DX`+7cpcEP1!HHvrRn_ROA)o-wGMZeKTkAJQIk6S;fKwgmbVE@h!{vuNTHD-+m+^GdN|j-UQiHg|~8!DQdD zo;oc3Tl}e*H3E%ls=0ptgKhiR6>+e0Vf{jQXV?fbi>6bLk)&yOe?bNEkAOR_V0Sj zqzTV91xWLoal$wsbEuTBl^&$9Ug)L|y7O+ew|}!$f6r$4^;w?Q`~T@Uj+;iguz`p@ zno-Vr;zpU~h9--^VqCrlpBwoI(%&bk_eb%~1GY0x#@k;Gsp&FA^efw$d{7iaYUQUh z<+Pw1R3@EcGjUzgrX|T$DQL<9?q_U3h#{`I%%LrD?-pQ#jZk=1%-uCKe4LZkd#OpXHu{$ZFCD=N zy8~H?U0SjIxRz`Z(y4QzkUYDvIut_DQZDQe?y*_#J;AEpWL)w(o|KQ#7cJSnjp zERs$+&>V-@s6U*869ZwvXC&jdj4aik3R&fKA9)Or4Z<`{MR5`}E-F@j#5klu`()!T z!P6~?&~&-DZItN^H4`T$mIFdRFt4P^SQD0dHBxN4R$j=z0Sbg2$--DStg@nv&}|!* zc&-B5PDTqMp_T;%dwl>@y+VkkYbEVvTxsltzLtB=xQ^7;n6D6*DzO6@`{#uXXh^SN zqSUs0_aea|nZdMXv_+sIPRt!jBPdU?vNFd6VVHW(OeySX8-%%`#jVy64MmkLUAaBq6dOs-%Uv^ZNIY3=m5VjJ~Q?iZUoo|Wyv@m zfidG4!=5qkIA&j)M9y4zU!bzI4pIdpAdcb4!(q-i9+)rV*poZsh~?2Z;y5*qArBvp z#vQY6K3u*bsYn@@Zqp)mq?BX>V!tIKbMnRzWxQN;p=4H-2s{QE-ZW`?E%mVsdWa#h zc8>uSj{sZu?m?nFMDoq@VhPYZArdED4H}lNT=G&IS-&ijoyeny^#f!oc!1+pr?g ztl9=c+MKsF6{%?vo3H{13Hwm8LZ8Kp>r0Xr{EiihocuAAx^6lO^?LrJ%oFgX_&{ZN z+>&PDUAcPNxJq2&kc#RoEOU)-ke~^qa4W0A_`GcOmqZp?=k7~{tp`;7SZNic@EQnX zTdsq{boUF5u3awu(a6CynX)Ukw+_lS;Nq^k2jx^5Ya+PHhq}w)@Qw{tERc>|)rUsf zRcvaYBs9;npz}RyuZ2w6Rkg^tT3?Sg`ggsI!B$p^&8)c+<%88G|N=G9)>0RVHsV61Qm8+F^y{^ zv^xT!8z@!Gu^llR??h2#BQg0@5GgExGeWb>ey6S%5T0$p%moOc-Q;Zf?42$9M^RhN z5NUudeKJ(~svedRS%$TLINq2%!i%yj_@XMqC_AQ?nLmfYvutc-dqyDFXw@Ca*vxEj z;$5={3$?qryM(b83P2vf9|Lt;V8Sw~pyMLY<7SgSG%XCDOb~S=5+q=r`--)xychSh zNH4B>^T(PZS^c~(l-i$({+%wMEk5Qj1RP$w-m#z33)W>|Gi9*bo~$5Bbb7f$<~yPW zxY!;}HMEokp-xV<$*J~vIMpuEsirw;#uXIRY0w*6hAM2O8|$}tSyMU`Z`)~Ko+Lgb z$w+f)>r-r%DbA(oocur@5P&tHz;vwnxK1I#X5Q!=^gNKc95CzQ0AYs;A(1nxGwcwq z&QOcj_zIF`P7@Za##Il;SR>5K@f$^;x|0=8W1enTV5+N%mZl5-hQdF;*8A2|VXJF! zMYJ$iGAw4oij5+2dDe)_JGy&1pu5{K!YxHC!i0+7HOFo#JZCD`$Wj-sj!1>CN9}-x zYmGbm2+NH#P!kZZmeHpeU1*;i(BOmY=|&uoKYSS@>U!7i!!u`zZMqK|qD9Ot92P#> z0!9qfC=sM7W<-FEaUXil!5HUJg!|BwETWvVG44Z8zK9|dQ4&Z)%7_5_S-THCrg>#N ziMy`Eh)wYA+py1A9KwkE^SWNIzG}vk7;{&PhWqgNcs@VIz>ivCsLJK+w}gpArmSubIVt@S zP<@PuJy^W8IfF-+uw&y_K3;UnARus=z2uQ8jP~hf;zG$bxh!b0C8;ruZAa9<@CslY zT=Wm-m-6C}EXEB>d9@i)NCTetcjO+5)SBYAHl_j}3p;U5QFEHyqL$>g6C0~|dA3JK z#t*(>bjTKnrf3L+kSWvqZ2{q&8s=Ygp+?-*69eQ_%ZYD5OnCI;Bhf_w_>b(hmT`j4 zs39&H&l{sTVk_a#PSZxJloIqzQ%+_{GMUgUqFf$nN-=;uuk2m5jzEn<`im(2fmt+D zVS_i#^Nc4@5Fa#EIA(`7H&D~}1K(5|QYnq(QkZ9gP|b~T6TG*z{Gm7s7XYxSzYoa- z*KQ|kQxIg7TJ=&@plp~Gp!^$zD3{>gcjBgU714Jtqu^QmZL|ff-62%M1fqSE>+?y< zI(teu!_3V_yhU-hBlmcdy?K?Py`Z?>S0p z5Mv+_WTyY0HTGB!--0bMV5R@oXjq_Mn{=3zWvqk@7Us(KWAw~M-f86gL6C11I$HbP zGt1|Mo;I>W6{@d-u@v)TGUmV+l~8A7acV;Z`vJ=(5n4fT-5bz~R4~24P{*c_8=?fw z)mlQYyM*R6;bowyTtSmt{VMqEV0vdHIW^(U{;@Sve9Cnpt5v`JM%~*1Fgcy~rrvkQ2oid1X-=|8gA`m(dzoG-Ji2Yab=B z7MMqrhkWj)0KeXW&Pkp+wCenGKROf;hh=*UFp)|xuH)kBQu*3E-WgM%I=J{uij`k) ztG=O{9SFjO*2B|^X2=L|^1UEmTwfwszFL>Fdj_6e$`!`gyOa+= zcvymGZw|uL`()w1|D9gAe)IRU-+A2*e*a!`oI*bk(IUjW;m38m=56nUC1aIT!42HhKb%%-L-dS~4*huxztog)B`|-6goE^q!o6wGfLR zoRLyWpgTYdgD@`yVIR1HN+`Iua5NNy&7FXvKdCXn7QP81sx&C{DaEMZE~VtWQX&3I{JqA{X2JXd`J8!E7SSiur0R9RFBoD?OC zLf$grQWG>cxCse?#6|_+$=C>12;Gn*SyQoMbt?CUrhp@VaT>soHN&0;U1|xNmvAf# zEs!{m*AK$gG(8*cc5JthXT6MvjPO%zSV>b_(ovDwZ>Z~>Wg=c+!@NkCxSUO=3CNs7 zT8Xz|aL%wtP)n(tleaR`(~1GvASy$bHLOwQzJ|n-r6jFD8jop9k0aVLtEIb9J9<+m z{wH;mE$l^?tU2tw)Bu?%dBJR51C8Kjqbq@k-Q4nF!?dQ^D^{vHI1~AqCX7aUF_s%( zc5n=XP^mFibSnmUh)Np1q`B|V;>Wn?upitTna26^TCLmam+RyUOzu%&-ogi>+zuOn zQnq{Ck~O{C-_l%5uQS(a2BtKF&&{63xI`i%3hfx_X^Y8MwL{aW>u9%7tMZR~*N}P> zx8Wj#BHB~g!MG+t&&jVE0*(=eokoudIVV=dpPK`*wWFGk$2hD>>w_Sfb!(|6aVXB2 zXeECwi5Fl}w<(hvp$nO1tbpw@+J77xG^&_D#}#yaojLQ5Q&j}JkAi9CGpewfq>W1e zvzLuZwBL+#9M9LTqs}UwTM`2A?RXE4?hO@(8-fG(-20&&{{V_u}?cdXV0+ zrLolbKQ-Y;L9Kaj13og}2JpEHWQ_uBBzv!3q)BJ1<6(U}Ra^)|xic%j5>DqzA|g7* z#&kd=EyRPxJ>8;47gv|?vegS$@INfKk9HE7}Z33y-njpP$&5LxF)1us>jpoeFJ{qu}HW~TS=$Me9Rzm)0oHuNp z^_UqqtBXy5(~Dkbeaeru%9tEYI~uF>TW?s4%O|?Z_?Qi--oXRajr|7)Q|SH4Mh15r z7C=p!wDJ(SNt0UD41|hZ0ckW&mwpnQlXb!VpucuTc)x|#ACw3wW+qwRId6>IZ6k=$ zra(oUWM_mnn5_@~;_A{6=@TIZl0AGjPH;m@TBmY@V4U<}^T}g5tXG!Fibq_enjZjL z1Ve_Zl7vNGC1W=9J0aIBz--TkmdTnaqGS}h6T*cmX_nQ!x(xzIkT{Jm$rnW9NVu1M zZ!DmBMvaMTS&0Ng9eKGO6QsAJD1cQ%p_Ps=(UVYH7ve~&MI^AyZIO$vZRc4~lub3k zuVq|BVR$inuEZq6i2}=CI72?3Pv|@xHy09(Li;czY0c zN^YO96rz#G%!F?+gakb41&h3#8?Dq)XoI`7H6fhK)SK|mLB>f5zarm*58Z===G82d z_llJF(pkMw`incRrJ+XZtfdYZxLUl_N0|)u5qA&)+?l@X_bkiwe?t^|!1NthJzojT z?&r#yCQWk9Nm|G}j}??OuAjD74`Cq19a9htWRreE?HT;BVuBU41jnbT;K`SS?o73D z753f;9}UpSY>#v#rS^{9Kmc>1#sGVHBaaITL!FXuO$&I(UtC^xk9{Cc2tpeI+6T~c zV@w2da&nR7YwydEjK$F=m%7^A6@T7Q5MLBhskatvpBcn1A=bcPmpN(TlP2D^yh#)9 z8b1e3+@oQ8FV#tBP2-}s=)Es6c`Hj3r~r~RCF5+MWJgP*Ks)d0b}m?H>QPyrz&rch zKNWPK2)RFuTIO9mOBHOztUYF3N@S^Zyav2~tW1P}{yStGk0N4jXXoUa*`7K6qMJcr z(XchePFYELrfOb#)6Ku5fsWvXW46`fhd+DkmP>>iZ<(s|OTFM{KOo;d5bdz)a=TtG z$vc+FJ60e9y!wz+p|})ZQ=Y|oge4kIz-5mGEY=Ei#rD2-D(7a%G zTvl=WVA2?>5+w{(8eM(OSFW?G4ONEFn<92W( zSavnOgk0n8_{>vD6@c_xFWQX>=#*B9SabeU#n^{D|{qWmEF2ROZ zmCQ?-H#A}Nm1tC5Jl{}3S8V3M&g$f#VE1KzjC58Imx@2#kDu@TcxTHzv~CD;=J>;P zd4;<-q){Z-es7FSE;qa^nGJJxHv{5R(#ZOH{xCEF94Hvq%s#kYo1Ovyn)W>pWMQ<2 z7PST8n;csqz}=pi96akd7`+1tL~0WDAz>^vlmGn4&8Ww*1VCI(iUvO07foe8x?mak zd&P<^(Q={93W<&3Nu*FbHMuLsTNk4Vj1^PBP50`9mW`p>)n58_8%VP_A#6o7%DHXy zB~}a;ZOE{(ZZ$nF-oA|2qeg@=k{wH-IrpwbL_;LFeGUdt3Kur&to7uUZR3rD!~|ZO zR>2l5W*juyn~eZDT++8W#v&(uvef^1 z1sNnG@lubMlBu_{g!eznA>r~5I3nZU)UDTwE-rJAAIlr695}ySv8~ zFq6dTS@%Ni4Ic@yV}9nqWzjgn2b-$%fe)nm><)wj_EU%;acLpY%#W6A6%lCF92-`w zpgLR<)&8m;BY6~#J@I;6_`Mxt%!@dN8(=k`wn!5%A&*s3$FCd&q1TY;^l>!XJEIu)UlztraJ$vg$uYq|1(05<%OD)3!=DKWz2_R&3fMFqQUAJX+7!h6~Chy!+b_6 zT+cVT+>t-ysRcH9)gpn*?K`?<+oQ8KU5bWybSaMN`kxINDq%+JIhSw?2ZvS3gm%?YACr8mxMKn zAhcVG9I*MlpuCLT=mZuBcUv=)E!6_*B{D}|OLJzsO_=@uTXnKg6iiu|gz25|Wa$4@ zcWD{!$5)R!Oru~bml&ekxd|gAm^HMtwqbiE$*Url%Jh9|-y`{diA!ZfSaDV4RvB*+$5KJ zpG?I7s~LFy%OxucJiEoVn{q{cCY20N8oqRfXV)FWIVL;KMCej-#qipyl9ucY`ynnf z+cqVd32kJ`mnMSn+H;1UVKE6)55h0P90k^H@54D!@xL{}OQKJr9nIJsN4YShAy|uNPFew*}A|r8Xl9$k@Sq<(2P%p2{WpIq5zNs~450!lR9%2;`<$XcqBY-qCP z!j;GruAKHj{zNW&MU@z{^~lz=$EG)}LC?CmD_;A$W$l}xGB40ht^I!8bCu7;j9w+cjN0_0_l5Y4gHFhSTqcZKFD%S6yH=?Nd=RY z%C%v>|}0WU%vlXZ_bnJuJk@x&h3pxR&qRO*b|^xSb?S2T}(V2LTVl4sS5 z_m;-`sQvKk*4~wZBH!Vo^g7`m$|Ck_QU7_00XLoP=}F3RCI;$xNkP7@@f0`Y;Jr6} zyngyt&BwN*Sc7AD8tz(VsUqw{S)FLDs=r5-P%^D@t{EC}AM|jD-PgaX*lu z-10EkSDL%f_&H9)vLN`|&L-ANHPspBUW?Z= zNbNZz6;J;#WEC4F@(da;_Hh?JcKi1w6zq*_8)TksSv1U#M#%Ck75&zX9t(!Kvxb}6 zQg-$O#<(7!Zp9t9iJYXUgy=uCDAA0TRIzf%9M_c3>UT8t@b8HE6ZoH4O6UN|FtumN z?Ds~@6AdU!S-6d{?S{hi*M`@^xAh(uEE%YN9EJU6M&EvW`Eu+w>36Ik2>)(wRfNY6 z+&uj6m9K(9iGQ~_2G(J#9K1!Vgw-a~mt5V(wc*-%t$DTn^x_pUzeZ{<-FCtuLt#Hy zSyzldu&2!`VYMqIw-lUG7j1*@9^ zwLUY1!;xAGd4;v+)pM(#1ObmUVYbbbLD&zC4Z)8^*-P6t(IGA4aqOizElNZBLHKi7a472^Az`wy>C?Fk93L26VQMMfS*p zo4@mPl<0Dd&wDJ@H!oeitesT@YRR8oyqc45jVfcsjHiMuc0AIo3brX;kmxy|OL(Y9DWDsP1Qz zk{%9S0JR3_)u_(Vrp1=B^`Hz(J9m>hfA{L*<(pUYO?n84Vj|5Y7vT~Otv=fJ6^lFe z2d|TtslODKncO~>GMAZLb@yN%u@S$-VB{H{=^Z{?ttr=IG@5d4H>6&fljh5HbT+|H zzVHVY3fsFe&Dai|d$1}DH83NZ_th_2(er+yOudZpxf203bUFV_~Fx-!~ z-`qgsUho$mtmx>81W$S9u(SI+nrSCVdOlfRg$Iil&I|{INkCpLpdFs{H{p zm#akaID!k0HmI#2m)9?@FPnamJ>#b%0;q!ua-UzMkD0=19!6G8)-=DU%JoaGpa^5U zfU@(ZmxFq_qce&gJSm%X;4g`CCDYRBHRJger|9nOSyEk*<_I>g@V^)N2AKCAp zN4GDpTNT=luf|rCNFi7<)HFBlNd6vA_W0jL%TkI<{l68>aQogqgqg3o{qeJFyk;j_ zAH!Q`FR9c(v==P@}clRW6(d+;2XtP;Msg7dt=<9O|IU_w)R zU?(78b-YE(JnzoOpZ^v7jlAXxLUV@t(Zj=BnYOSXxm{_cljnb>X<9r-9=tS?R56$^ zZd)->;k?)<l(vrH7zxl9FT8-Afz?dh?mo**x1iJHrA%MAz_L}JpUy~_ueIPnCXH44qvTN z^v$DTL8uQy-yTHK=YQ{;>P$1c-oN>qysx1kbn%)O&JR^FX-(i4#w4MaZ?FIQ`r?nT zUcVuPv`6RY>E+A4FQzQYds&Pr($RdS6+vVE)=nbr*G4|AyLSzDuhnm_?+%6~ zVyCo@ynmP{nBWR9&!5A|1!POLpg3-#m zOOH)O>}>X?B!&DNlnRX_BKB!C-eZ%hEYqhVh2I9t6jtaDx5pMu8=7Mqc#_I{aZigh z&@4JC%>fZ{s4QkP(V>0^g6qJitN+J>m!|f!sSxey+F@6TsGX9UjA)W!zT(-%1d&Ne zGuzsdmG_K=rO78xo}lgV$yZ-}MW8`g$`Zb510nd-ub)1hlmB<|CKS?7OuJl#6{~PY za3DTlbw>XE`kS|PomBJJAYfC8UgL<{&N+GMh2QzMK3c8^J1jxD5b%WRt&neJLPkQ`~jz-5SV@-@DB<)Gp*m;y32;h~unD>$_Q z|A_R1jacfkc(?h0$0ojKRD#jjH$|n&KUN~mA_~VgR(tef2Li_Sn$!&2&N{08us);r zbEu44*6`lfEp2x{76H|m&h>qwg3zjzGblrEgRSS}2SK2&@)^`UjwxhrB&gl_c&G2z zQjV#0`-A9!=Vq_3pK%l|Y*($=sS3JbXPVmD0>{m2fr_sLE6^Sj*EEW@dbYt9{sm=d z&13YfSA?!s&|Ty%=9Gjc6dC`qu3@-lR})+#U@ZpE33;Vt z2^IY`WOG2u6$Xc3lG!4YNw}NOsSEf(cDOXq~V(3f$^6Iw#TJDZdv=;wVSj?i9fnz9 zw?@dm3XA(>bVED9v+EK4C0q0fUj-3w_j0K*P3@pz4l?XE@)PurP(>K<+(?MrUOgA5 z#>%=W0X!6YOSPZkU15;W}KLl2uAyVfBE3t0R5w1f>$jCT?M%qN6tNZaz0X5&Ijttc|dc#{d#jgT6fMz>d*NY9Xb!`(Rsju?bE08 z@j7)rR3|W9Osv>^z`r=bn8#55V>PdUx*Ay>o2;&b>N#?$yI{pFP;4 zkLRJCJoh~1N9*SKX#G4NxufS}^z@AH>Ny6a>HqlveLeTM7o%S|th;Ai+&GR-o!27a zURE=LrXy3O+Gv$Ms;Q^OMncwD(NOK@RDHeaK z@^}^B3*pzyboVe$)Tu^aaU4sX=^f=6U1V0zhfpW9@j>#j4X*0?SnRx&tI!}M&>E;u z;eW6`^H@UTCf8|yd~3{)O}rRC-%=V=&5x{*T}GcVs5az#t;bL}aXlJ%_6{_UJL5ir zR!m;iu0qP_w%Za6r0D!xz)BzGPSuC?saWGY^Y#2;jAu8dPzqK#S(-xorXgj=#3P~G zbk9oEO-dVf-iJT0c%d{Hpxgd>6c0oP`5T&#bSb&_{5}qZE@V~O>{$GJ#CAai^io>y z+XEY?N2n&vR8|N*XvV&r5k`}>7Qtp97C5bB`BE>|`O4aSXT+`%4_Uo{3FCKh zF;>b18O1^b8B=%>z}5z5?wb7>)*ma!?RKcnxYE+4}cJu2aS zW6XX+UoA-CSIiVwrq-_KC8@7@5=*f-Hs&=DEhf0bK8r2tP!Gk*D#z(Ui2=P@rW7`0 zCL3DZveZddv8QyUH_w?T3d{5$3Xg6Z1vcXixySSQG4johqMT7MM%fk`SNILh6YO;7mt=aYn6h0F?`p8U-^dplTFE7ko7kV6}%OxhELQ3%D8u)CY1~u$^T^g@m<7 zydH+586E1?9W5G~*(i^&cO;;V<{t5!lQ)LpjYJ-*Wnd+r!3$BNrnzAldWk28CXLi|Il;WoJb;_7mwHNktc z$DZIl@!47wF43ak1u&79^|HPjBmP8d^!qDMJ5&2wv&uS-Bbsga2U<##vQRVMx;bdP zQ^@2G1EG0Dg$&>i6gz5&iHo50Kin12ZbsZ4}5MOAMUV zD2T(B7~i@w3Ut5{6P+)&3o|NzZnQ-zY>6RBlv}wTKQBv+C*D}D*|5rtH+1+F-mGb+ zuDu&k*P=a)GshWaWYK7_b$mRZACEa5*So-|ZB@CP{gyD1$kgTlEqO6=l3GZwZ2-PS z3&^Ci^^CtUxV|w9+>5o^jO?!u7MN=>*;W0IsIt`_t|f_)m+Or2jBmKf2Git}sgZt$ z69h0%^6Jb0n{U`g7TYriQ2%ixW25X`-`*?4 zx)~K@L4s?l;?Ks+H6lt4Fq|~dOlM7O?gy-Cp zNuc;UVx1{`97}I`saUo|sfdcYY;xf##YQC0Y$mziV?MzRhE*1{muBJV+lU4 z|Hhk5BWeh?*ezk)L}|`V@0rJ-Q|v>|3qba3mPnEAW4PLZQHxCA4vRLiq*dWk)$eG= zQ(kUeL>OK#t$PesX0L}uOjVybp?O|Nnyi`fW_~u}v`q{-;nS5V;JPD_cd+)6-U*7%l7jz{ThD~5HAM->rU=I%2|vyXgm zSd-?47sQR?xH$th)#Hyc8p=tO{8LixM=}0#92F z*?JV9qP70fHbesRcJQPC&fpO}T7!$O-W2vkVw(QTJv#UWAwE#s;DW8VDvL2Yf-f|Y z)09e*1=&Uo2==?}8Wmawu|^Oi&T@oi@AZ!sA5V3vAWK$co=F(ZWg+ zVKVw7O>UV;$+yaGPP!FzV;bM)g}mb$Ia%6*JrF62rfICpY)hx3Sas-5Rcb`TNd52- z{ZVN8FZf-;Vq{Sb^Uy`y1zR`mEW;kBcPmT*U*q@1%AlDhD`9=ib|z^diYave=nakS z)#-g8NKOa&Y_W*zWOH0B(k2aiM^mZDAE8v)O9n!_&ue@4rQO1Jw^eUvgHiT@Zt{$& z`R#AP)@JV1d9g^(-@Uqc`R3JplOED!a%2;&<8TMNMsw#duhUzxl@+*Yvh|rkISoijTpv90%4}{}qWlQm@D6P)TP(tQHkwe^@}8uU+(XF0oz~sJ ztFNpz#XHPpA;Vjp$W30z4Of<}n09Ul_u%9kx*CCw@VGNNz}>{3Or^3o3-_qo_RF=r zpOtb}DK_K5*te(!-?A+P0!A;{ywK47N(n|@_+1Md@BaAmTQy`i^2F+MRwm~O%h%@y z`u@b|fwyNP1utb0-;U_mzJXjYky&PX#mFUj!Sb~OIlAlh1Ft_KL%5T{rSRMh)K(CC3%vbP%!hKarCZt&~%U;k*^G~8!BDOfp>0@7}^ zZe9WX+;cS2s$4Ts@&qlrYB=D~Ctc)niCRUyC|bw2q>4gNC^MoQtFM#bZzy8w3)5*&&cL-SC0@=ud(~4y$A!SM~nY zGSz)GGL_JbU49d#m6>OoirKaK(d*AVpoaAI2>**i9wKDg?R4`8uO3!r*_Lr(Eg(lI!c&;7O^3(FKCL0KNr7e_ci?5{1L>DiXR;jlaEj2p6es z2uQYQz$;0!uIsd6EsXZJg&=y-J63KgLpNR z?cd=ldg~?N*+n@t9pfjmxn}0Dt=PGys0F#>9*!gjPGv-tQ*bU^7o=m`wr$(aiEZ1q zZQHi(lf1EQpV;Qq|IJj*&91ff)77;ux|$KD^hTh&V*)C^FTew?=Qk?R!&vhjn&Pi} zmX3D3+t*BIMqv9fOGEP?r0k9v$WOw%&xp>VUc4Mn{B6bQ^Vyr|YTuQl8M7uFHmW-M zEXA3=&^n^&Oc4z~Xl>!ax?y!hrCOS5t~BZDXm~!ahaAR%+?0&oKnT%oxA}TKj8;RU zSmSimv(gdMA*lPHYE}FV)e--3b@Xb{mWr3@^yr!RXjmq^Ck9v3LIViac{}^}L4S#p zyoBk|n;8zmSyIH2XP0S0HXmQH+0FuZf>7)f>i!&10Z(1O_XE(ZJ)@7cX_$b_da}E(FEi^$1I|4Q8}WTOrzueF7ZRZw9dnh_=0M=OS z#w_zEk1B0*))5}C6C7i_o#5*r9yUN&Yx>AJ13(7+mmEmUf--@+9D7=NcZM}Sa6KuC zFH;W>coiE&;si|+wYzgW_@MS6aykT3mLhVB4L{33+af-^Ea(0jaSUro5}PId(t~G) zwcUQX(9`IU`7-&UY?qGE&t?5p|;oxS_Ybx*e=xAMk<))JS{Y5>*kAx|Lm z_KCPeI)H~!|KsuLuu8&C-X<3G5ZE|d_pj?X(8cA=-slRsGz>)62%E|}TDRKVn5_p= zbrMFMoa9up-9wN-XA0_Vi^m~Yyn)OMM> z0S&rxDV(~l2ha~0pk0E`kVWCkRZ5nGP5cqfIQ*gOw7&iPw*y>{a4JZcO^w7pEm2GL z8WC}L?arDZtB+XSf*$<>X-Wg8<^yqJ*rS>I4CGMgBuZ^XQ}Akuuqee&^D<#%p#fzS z8m0xyadksbQKR=&jb1|=Z-AK^(Xnze>0<@y3~dLC?(UX$`DOlZV@~zaG#-Z-R+fqt zybJ5u;{fO^eg=KU?>Nd)-Xwd&H$$fJYA?tSuKdLP4bD$%!?tBLo*_hqJjFY;DWGwi z%e3fQYbtK`wcy^==iS+N8mQl*E5Z35Z}rA$DCgH{(_Y8$p5 zYj(0+{rJQyW`iPcdGyu2_B`L->;z)4Z@_&wZ~`qVRJu2TD-uXOOqbC~wwj*p5I85}P;%7%F|{WZ9o0V_v7pa;H=MmQoteE^%Y7;W9{PqIT~UM{|t%=|nZ%$wrIn zHkyaJKefxTu1QKxxaLPm-bMaI)DAE-sp)uEa<7_K>_XYxf9zMxyF|UOW{KM~Wb0WZ z&`EKHMqj0S)9L*F#zNHO;ae%t1j{88^Qpr--o*66Lkc-?E zUFhi+4hoUv)+8s;JR45|SRPPBYHfqRWkSF~cHIQdlA22rlQ){>PX7H#f)8{hE?Tv- zS($!5apCc|8;820e7~%_4UfAnzEG~8hV`KsfeR+QV`}zF*MjCR=2l#+pl3z(=v_b= zG_se@#3D398B>6*ryF!|L4X)K9My{NFBM(#+_;Wc9p(#>{=imEuIMi>~Zv!rS|pjxQM4Gro zt_DOcf3cZ^Cmr0NP7|4h4z9->BHmJz{Sze;iJz=qqj_&0(O1PTvXTQ<{Va^Ozp6PI zhJv;ZhkKA8wv$M(StQ$Ui^&@H)dxh1CJO~n76lE`LV+2z{ZwN1y>a7*8!x)yShV(> z%Li?16<{f&pm7zRp_Oo5+?2)N!q2Ax0|(g8M^3pRz*?65fbS z`{y#EV75S5jDPF+YEFoG+mAC%y3J^Ksz6Wrl@xz*`cUKlZaI$465FEq%SPlisn$UE z3ThQ~Ni$Uc{f%L6;vVDM#QDPKr4{L|>oeNd9~EI}p&J#wVh+uK%ZC%ihV1R;>t~=_{QaKQkA{uy5$$itjcd}T zP5yOTvI9c%h5{BFztGyEalaUo^doHY#vvoyNTcK?{Ic*&lT2m_=21Va!*GA52ux|H zT`b0C3)AYU{Gc@r)Rk^$<;-hTg&(ZmtHND5pA2?n5@QN+$b3rKn(EmRpq#xWf* z11Tj878^B7F)c9a!l+B|RewC~S&8aJxe7~cj5(>0VZxf#5n`PrNC?>$4D=e8PQ1Zv z@d9KRcv;x;;i2DfR8Yya`e(rDJ-Bry+E>oGb>zPnAb)$xe@Cwd&=EMc%EfhQ0`VU8~V4x3)}5=OgAy z7ZsGY?R-X-F0Q>l@R}_(&HFLcsbRxZRW73%tlDHY`pu=aFzyO20m)AG^$vao3$@@&_X6Y>y=7+n;0^Rl}|hXaTx_6?jbVhW2t% zM-3X_Y)adBGzYQI3c9&t^^$vr=5P}ukstUj`IBJj(>sChJ>lQoWo z;qy0Xx6JP&zh*;CL>2fnMK^4PXR^6*@v&%U&qN zSssHfi;sjQcRv?dve)IOh*IsnI%n!X-~aYE7U7Z^CqC;NdT%x8(nd3FW%C7iNn>CH zKmFse*cz<<2K=n~dN>3Z{=RD2K$hS>lfV>9V&4ctj5kDb3*AXx% zt1(rRH0|T_U3g%6I(}Upp76JH+21*EX7vf`LBJWv`yaFk*D3}KJ?$`HbyvRTigOh? zX5_mIg!`<&Q+~fQoc0>X7|VW7cG3t#_LyzOlU}`lFhG6nhqw3e{ZpN40iFKRlTWX8 z7aC3f6Z_D%A&@4el_dP=Ynq`ax0(q*C=*_h(;{NPlRg_o9iGyZ%)MXlTj^74!P;YX zV=VYCC&f&w)pP)&Tp%oEBddd7bBSg@F5P9`ON6c0D3=6JkE)Uazf7Z*D^MD09waet z@uH-ox0tg@0DF@W2J)&>XO%>qx#ny11M;}_k z_2f3G^^iG*C`-m$vG=Zu{81vjH#^`~sPXDDgZgn!mfJu27nU2yD6&s?L{vR%cX_qW zR%ffDH=<*s8UhY?^gfQjzr@-KAijBa8%tkx)f{$AHEue+xHL2=7 z-M$&}Z7hXSG2i$3(%>i{AmrEY;Y#-HwuQ0(?Qm+3?>(c!;ODf)K=A#c#vtJN*dU+p z`=X|y{|ocs7`UIGi$-nn&>)}B2wd@D(51R`%548u(XsQ_pW-NEwoGOIle^oEW)wyx zAA*mMU17R$I){!YpvQ;h_hREK>iT)>d76%T(Pc?1BW8s!2|Pt6;G}^!^EGC}_9cer zX`j1@+k#Mrg6MjP9mW46Mu%bX-?4a^OqZ8MCAR5e@WC18YGk!E4!VdjQv$IEG>-(N zTVDF01^==3ot6OL5x3{A+8@ujN|KN!t1UP1y|G$w-@|ynGv?uUPWEQRMh_+s)7zqW zI*8r-sJ-Xr7l4`nkMV6};@1J;gz)!zZ~UG&T`K7=whU15%kF?>cwKP?`u&Ob%Z)j{ z_Q?MFTA9FI0E0%(o^GbLD8CEVZ5`}_(6cJ+cMEQ&sU%ZF+32rOR|6I%NdntYHF1N$ zCMK5)Nc&aMcZ4o$8DAPZ_JyVGcaK1)791Ue;su@`j=j~mpI=<0F3X9 z;Gwh4ces=2kaGazEpe(`q5lo);?=f@*dmUKpwUKKKXplwF)s`my3ak7iEE|LWIz6!*&EP)5qy2dS5+KmFKz%i_XBhI;^`}m5G8Kg1DDEXKRtrG9tky-(b za`tqSr9HFz)Zb)Wn=hMJ9j>01Toy_ZMW;-cv}l9@CL5>au@@*1mJkWxKPyqw+S`(%-I1} zbr6(@|0U{$(bQzckj5#&SqoZhLhMqV=DJEfgw1!;;G`qhLW|u^rZGTgGpnXvpPe8N zuPgW1?gW8#2WuWO6nNXWX#l2-!X3&zU+fG_R`y3j8w;=lAHYT?l3rue`$ac}i zA(>(F9)!Z8W z5yGSz7K4HvO%Y?;pDoIDh##8mF{t~HOwkKhpWx>VB@`rDcE$p z&yG^-s;`^6X2v3h3?Hc^@2q2!P|N(nR7A^sc~TOtV(C2f)-fLfS$h5kbtP}ff_BTy zpwXKS%Qa)^(tC(?BexN+y)tR@frCUI7X}W^DbX*w_4*E%m%v{p?#5aOsY2_# zm&iElp2Dmfv9dOnlI%f)hhiPL-b6t9%YA1+H2E07U}x;n(Vck)#cr$tlD8Zw9rw0T zg)9p-bS`wgoBgW6q6WATcbqLpZ#AZ>hCA68@&MeCqq3M`_NiQCrvDJaF1_$GRXFO~ zF!&5VN5GzRF2XX>#@VgijAYOf@60s_G?v1cwY!&L5>uHU8-F4FL3xBkox!zs2AjD{ z%BRtGZW9x@dYMr5uw>p5x*CZ3iI9!zs_7Mzx1?vME%HRV8|7sPQWI%bs$j8#TwY=U z8_z+hwqq9(DBcc>5?w2Bw>qb_bgXIRXyGim&Glj(ke_$G5#>gsx?@l{T4*n zj^c3xU;}lX*0k|kXgBhZ4jCx~fF}=48pwl7W1&~puN@kROhn87ML-fGOWB0JJSrcm zA5wY<%>_z+c+eao&za~Yypq_T!#;Mp zOB&wACf8TlQXk&~DkKg2?Kl;2;XRf?PtFU)|1~zN%&Owp+htm?EH(%xE+-mz^#(o# z@oAZqiMzmizB>VeBQ8v~`z|t|sFQ zM6dnmUaakSlWk57kug5@>leYH%{nSj*6g4g&01%C`P=(3n1(^7FH^odiSqL=_p6HU zlG*DeZVg#&dM9{{5Tf2^uA~#m5-Q}1pAkgK=J%jKsm_b@Y9?A2rcNd+&K3UPa2y*U zH(hNw6peL%Li1$$a}z32H;Z+}!c44H;rCc3^Pxc!((vGm%qDjjxKFg3V);h%9i6+c?ru0yRAu?eC6dxKN`GGyO(1 zrRhjCx~se5>b-{rnCwUsz$}LPcN5R4Eaw`F;XqfA`vO8t!~caaL5Cbl#2`fcpGsm- ztesVI5Cyj7|DH-0IhJ`cP*Yx|QK= z(XGO&jZ5T`HLaWcRG?Q;@P(5CcRWT03O+^#D$Zb;95kJ2S||;XZCW^=QV}2R@qPjU@sFVsl+z)DF|90 z;$A-^IbFGMW}OJryl3+?xKcKzHnUiBnjM@A%5FT}q@a?XoQuvXCN7<>TZ&KH+=I0F zxek|csw&A{=XYuyQR~2M~d1-|f+=+eY=8h}Z3OC3hF)=pgPJ z=^EXm`6&r8MUc@bnQdg9U$B5AT{UXJzo<${MaLy60dUY>oSKqe;@g7^v5jopgLmRF ze0i@`@CD|XKR7<5NC^q4_dJT#*GR^!64_?m*(H|1Y1bORt-;2b(;1pG+G1kcJd<9M z&eX7lORTj&$bDvdaBp_@$n`O8}(va6X3c0xgC zyCnv$tJ9dRH#@)n2kPKM$+LNCxx>6Gqg=}A}U3zO(nF6f4l#0|U+GD@9x*!gz@mEhmJwc_9WXqS{Cb{w~zRC=ql zQ9G*qrt&29PG)nrl21Cbra_d>vcJ>-98Qd8-HwKI3Oo4R%#;|p39js|;IvK%fsgv# zhAhkA_?(JG$)ZL$Mh_6z^oA-6SoT!bo4c$+Sf&e(e?HWFD)BY0oy8|YO1P|Zo<(+K zI)s$vPweLLq>t>BUqHKAdA*{F|U}4`#|lB zw|0(H%D0AJ4E&@3(Awh6-OIxXyfl~rP*zI$!Ye0Mz#@aUPET{5oU{keBQ=y}mRhR) zM*P-%Z28u;m-=f#F9JYwV@~K&s5NyHBzBbXD}P$nrv{8vshmxZXyeu3Pb~E@QSi`j zydL&`T`qK$*rs};AEJ<2UqzDkzS^j>txly~w}@7fh?5uoW}#6ykf2nSA6uTV zzrsB8Yw}Y0J$sEh1d)TxmoKDZCJ(o_qUmc;^*T41yBKzum$^K^*k^GHe{1-a&x(P4uveKGj;%^J6wTv*UwXl|G&D6U$!l$;u3f02y zj;6Pf?s;?+wUvOq0OhaK(0b*sRB3Yc!O?@srT{Q~9-rZ;y%oMMO|n1JYySuh$?xLM zu+ZsY23p3!WQwhC;LgN5VXL%r=G+!(9E-7T+S>T8rGJ1+&oR78JnrDPBvWxvyVHG% z<%NeKkfH#h7<-DPk=!%ZoiV5xz<_y`NJQk`@&J$igrPXZM1vIE$VG4!C^t|LK^7+R z=OqvA1f&+1P7iG_6DuE;W=MpS!E{o|jljdK5bs%egE75Sp#mEun z34;~wBe%avLQ4ju{&`X|Ix9SY3&u-hMjM4JX$kAy!3V=Z_1MK$5rFDqJmteu6o6`@ z+~vcP6{tn>7|jM){J;%I5IsW9ePCm9-xLo+9tfx|dzKQh1?P>$fI^BzCCpxVrPk&H zrqABDrOy;F=ETGWL1{MA;#i?tXmu>8P%V(o80zrL%R+>Bt?VMx5k$VQ=QkHNxDryL zSZQDZo>mY+W9IwIy*$SGP8*yv)7ezZ@DP~S5|Hr2aOOLB$x5tUa95eS5TGW`*l-CG zW=vgh6d6OuEN56LQ-$ml8NxPl3?TtH8N$BxfN{Qra2kvX4n9Y<(4c52y)IlO8WjzD zNg@XmB=e37_S6<;DEEQf_=4F&CX$FFrw2}nQ~ST-e#x`rbANUrAhaYh2~1o#zJ#U@ z-`_Z~*fgV3a_uMn-r`V3a^Ak9k$lvqGr3{VF*;2Tujd%g`1V zp)COAiDhU?w(>(0d=nZ04s^nt=mhyuz~PvxZ>@#gD4_h%r0G^}6hI=#j82G)rE<^A z`qLGYAfM+b(69=lLYZ42ZxB|6I=ie?M4&>MJC~%ZnCIxPfaY@%%DQmAF~|4w9e8;0h4 zWW5Sq8h<{wi$K*b1yD)0jz9~w_GcEkbM>hp4(^`cvlde-0NIx4%vR=e0T;S;%JDmu=W~IYTtu;=cV6l(tM1#=IC7$QUgE0+ z^_(%IcgEnV1oZ>~DnLAVP5v(-R8h=T>?j&po;hf_JY{T8)qnr6Xrbd?r< zHu+DH^yk9P5~+Mu!c9K7m)7!gxms5`_-|6Uim`SLfA^gM=_KB}GRDl7us>AJfIb`E zki-LUgX{z+T(3TCwP&&5>rZU20h4y)p~NoBJ^S4HCy}F8;%junO~0?8APgUt4UH^J zI*XDsPc1Y13k-LD6Z}ve(Hb4KW|zrGJnJfN7e34Eh4j$P<lC7Y(XgzEo0Uuj;>tcyy24or1lo!}ORHxfnv$(zpve zq?50VyMoH^?)TFZ&bbNk?Ce$RG~+26R=t>0f-GLr4&73xNz>6KK|H~y{3*OT*pk<8 zoMS*;I+jtC_vg&>qKJ7_u^sEGi23pO89GDh^m<{IpumosXFs7hZ8#)gzKhhr-g=Q3 zC>+N&7s{T+)m)c~0*&zLJE4u-+FIsae@=(OzqePI4Bu;lBhG(oh`(>9(bJGyN2af@+>Hv5b5toiAX{^LX` z;OlQO*9}I>H6`pVF3R6eKbIX4S+6PayIr=Zd!N_M&u}~Q7+^`N%rrI_ng;OAF)L9t zC(|}N6VtXn6VtXC^q-98O=T2h7u0skTpzmzwpn0&E8RIPt8J+x5tC9UQNbp1k1<%l zhm|}8*icnia4N(j*I|=GqBYFGH`cB6m;^L681U^V?2*6A&|2`ZuidsY7QKubgf4Ng zDVQvaIO|eUZ)$d?gM6I^RzkN9(zSU$#P&+{M2ML=)vwFUfhyx`XI;y*n*NQ!Z zZ)b0JcYn9%Db9a_U(WXreLtqyoz$Z4KyeommQ^L)P5csWBZLF2ZlIPUB!!}ddQp(O zPH2jx)-bMfATIC4Lw(QF6hl{SmVBtD*mv`{bZ7XzhboSsU{G&-BUcm*rQ5w~lw;kg z(tGc@wM=G8ro8^6e54h|`^huBgy+~e&4Q_z23;NPUQNLN)}0)s*?SNgk7^!UJpQc_ zXxt>U2#pzWHrqPH()pNf6frkn6&1{DV=(%}0n0lZ=5uzDf+M4bjV4!?d)o)S)WfPC z7pK;6;dc$nAP?ym2H*>PJ9@qyElunN{5)B@Jc&O)*1fu1eD^;1&-N9R{rY%4jV@3k zsx7Ed%n8lJ@6D`fjQ1YWL5fb-EEu_aBDwG_F$tejGNdcn!DU2<>Yxf^=3qq=MQNf2 zCFAVtqc+FMAHg#~sN}M~mfzOMHVyAGiy`Al(FWn(QQ6@^nj&?uTt~|z+K{snp);i~ z+{iJH+mZ$q?*&tr2*+~ckTKm^p2ps|=!<5uMH?+YtKrPIOg%jL+}W@tT9gD;RWwCa z3|OO*lTsmO)WpKA$R8HB6bKrltQBR5l@wPDY#VS2lb+*A|9inIKlkflb@8aW=Wt#r*@*#Pxi_6BHzAI7En7_w_PgD|0ZQ#77(kiYM zi<9_}8IuB3IB`V25GxbNDM4;{Y>I563i1Jv9G{jTABBcpPq9U(DqbSkqC*DG`L-SkAm$Xct=q(Psk3Zdrdfz_L8p<;@sT zuIq2}DUMaZf(M9}RV$OdDL9n4lCXTFbo}S@yRPAD4swo(9{{J&(2%V0`j>KsMT;*TObkVV`=d7DlRV zYf>(K6olU^A?EUSVLebX#Vc2yo~~Cy4<90Gd?&M7=rcdd^P+5+xa(eT$PE^66RI1!m4qF|TUUHf;@;!)>newADO-l8W z$Txr0ziAgZ$bz;7&c1qO6w=xIr+pzFR$uh4;dWHaHnC829xW3m5*c@6Q;@Dvqh~lv z5y#}at0tO}9j3S<)v4Wi$e&i0~ksiD>K_Hl=5{ z_^iJXtITw7Lg4w%vA>OwNeV9Z)%p&G-%`Aiezh_yhS&|8Z1JT4C|7EY->v+i->YBD zVAc`%AT{K;{l`2dJW|9e6qn%9(18%;h=`ERf8YPN}V!zPi!0L>Cd`dXZo67 zwWpVRnSlJo?sJa>0d8Aq5@3`0VSP-4AP5kCX|_|Ltj>quWe``J6fZ_^RZ8j}>sorB z*0+?D2&W;-&#b&%FaQY75}|i9kyXhpzz@8}cTPSf8(KwMCU*|`28>U1QbA#$D1|4- z8Zs*#D}Z0{eRqN03`VP{sa3ffF$Rt(mGy?B-LpstHlp(!nP573x6X9qr%ZqCw5>ZHFcJ$?yR}5r&M?jz=`t~39N9@a55{*N5Y8{;DzEL7h#@|#=P!H zdfgJ@*zye?;r=uE%?u-W;`({!y8Q^ui+`;}gn@~Nv+q&V)aljXpRFQruae|lzP(gP zfPTg&IR3mxiuwk7+h*;xAmjbrG5sfSz!%_|jS4J~O8zSK?j&lZ3AXrbF|tF8Xm6Rn zMfE3ql^G_rXyf~U>XYhEJje;vpLkOXG%7*=Y*f-e5UlkClxr6R-2BLqoMbK51>9=P znW)nx-l=fafQ^5VzEz)QGT@fPORr z{){|>SF@W)ZZ(jYnHt(9-$QS5&n{iPKIpPr-vJW1%@Vv4+7!h5aMtQhE(zBB%>v?4 zV~V$2S3&}eMi-atavTy5P66WQt3VLhzwkD|&UBp0NqG0-uu|Xi!b;4AA=z_sGrjk# zxrk&W?VpPHwaXl9=?Ekgmiviv87iR&;brlGHQ)P{@?WD=0Bm_GMzhgZX3>uU2FwTt z+$lO&(;>pve-GP8?Ng~JP==UE*3tEmAe2u&eZ))lVpg;&^;|sjolb%FURh<8 zNTzjJ)MC`MVJSIP{n8?vhVb+_)WOdozVb^P6g_#3Mqke#9p=HiU0B|)yMNAhbk^s> zREBxh9R#Z`yG&i^CdA`2=-J9AUcVY4>cj}~1{W*+U-t*?lx16W5QVcIU0uX~MfI0* z+o8nbpncB;Z6*dnQD!l?xkQ7zMZ)wA!1R#YmAAm*i(?$h`%=zZjP) zgov=Z>|&rQFSrSRF3%xV!r$iv6#@i4hE+`k_`Xg@au5ytI)5C0jNHWrxIPZ2(m$s8 zc~!C$L`&;gXtF{%QkAM&Jr95LR~8ujCh|1x_q0No0qt>Vx<;tu$(OKjL3F%pz2H}| zHqys!`nG39CrFF4b<$Ms}oIS1N+B z>n&DN7`~HP{5Fx4(#n@1{w)J-nfN;AZ)&2;cQ&5Z$PPa#l0ECMLITroYoWiDcfLE~ zQj}B6h1gZZp#|^R(VPye@KakNwh!A{EmHq84O?q|NF z<|Ao^Qbmj+J6Dk{fN>1|!Rl>r^yu@g81jND%o49cve|i2u;9}+96J?NN-f>!*4F(Nz<2VP9P21P<%06&3nT^OM zD#CzQWm53W9m-PwiRZLrfa(|44B5(@7plv;S0)}qm){H~{jasjM6Dj9-uzF*Mn~Yn2w*xCyVH%z9d-#1W{c8(5 z^qe{^&Q>#3NdjVV7KLaQJ^S1teSxn_`KPlbwl-?M=OF$X3%dC)$xQ1F&{|(R5*T9i zwo+9fDpP&Aex-v}X*L=_+X5|}^Wm@KkaRS9zNg3YU#Z-b(0|Bp5yIa&&1)E+cC|raV zf(6OP21Mv}O5$qsc0vxsP&C*ARu)<)7DJGK48@%s9$AFm2O7w28jzj|)ctlWtw_Oy z<9V+Sosa0$d#_B^hFA%ogJ{}m31N+5OWWTCLO?W-)cXf`z}8nt%?#{2+;%rTj@=w`2!@3kZqB#se+=Pfoo?ZiLU9|G~* zoT-+=!&m&K(|bFl+DWh!gVS)%k;nB9DoF~c{jKWI66UNocE}pHlIA6@iXU=(&M#QT zEVH-4H_M1i@&q9w!q5H8JK`@N*IO5B#0KH>r_f2u6bbrrIx^r#Ev<@FhPZXe~NC#Tk28Cm^lWLET7kBKjE1-?t3eFnopP zMP0^~wo-YORza+uJjFz#YzYeE(t=;6Ll6R`EG*}Wza1#}?&<@6bxp}Y$+;d^Eyh`D zzMWuGQu79>x;0W@)F6(-DojXG_`>FP z8^ghKRc~>gP~YdFK;WCHN+*W!hmC!RUeW;1&++%|>Fn(84&bos>&DOfn?>OFmX!7$ z_r4Y}LOAh0mn0M)-+yMKt)Bp4Rw^gonVDG^0Bq#1w6T`*UgkU%u|ey*1mB88Moeu4omG6xHto$Qf8XZP&e6y$BYV_O z(dSYAVGjCY(&fO8qg@g9ZxsASi@HLa>#V0ej_3^X!;V%_>`&#!!B-o7`^5e0F)6l6 zGw&E6H(BpUEHe3B=ru3;`^*d%Mi`@&auwSn6OTYD6uEfk_#{;cJL0L_88uGjO-HCJ!ZD)XI-L6U!h<(F;Z; zh^2S=HxxB+F)nS*4c~&sM&<-XYd)IcncLh{o`sfWT`s0qv0in1TOmHV6Yj&Cz#}T$XPaMh2QZSpS zYN}Q^jHg}nE(;N7#RmQExQVOusu*5L%qY1AE9tXaiiKe37?6-_kiI3n7)|LNN~y~~ zfnWh6fqjKeAq)E|TXYA)Sls{+scf3;fW1#BmTsa6igSjTh2z?&336pOg18acj5^yE zJ)dWoixW+nn<7hg+!E0V;A&wwN!m_T4ef;s(k6M^oGuNd<2*BLY2+D~Wb09yEu!8( zVnldqe-ZJTqKF{C8Q=nRjnd7<3c<^>)b?c zxwcq?lFh(c21q1_JYgxV>0u2i25nywR7o#w;SuQk_LW5+cMS zD@`DGpIiwIxhpKRRq<%}*^bEmm8uO>CA3Rg2(7jSb%ncE&PwI0wIx8{87P^Q_$wxS z0{euNy96 zqXEjh9pLB3{~29L-&{=GMFslDCQ8#W={BX`mDBSqhdyJHj10Gaww0mUb0iB-9s(x6 zIA#Owdr+4Dkvrcaho(UdB`p7JjvpFuyai6J<}2?AlQq&lDaJ-1Rvh;lcM#f@8vW1G ztYTd*$O&DxSiRb5Y;*@!IkU#<;O1TVdcRuBFYS?`&GeX&ZV|cE6CY=GI+zVc>|;0F z%VdG*m(03lAhS&?Z`Lwlt=>$QP4x~yk?ACXn!E6uh?WL!!BGpE3G3PjrXeL{yrq}- z)Xc|5LN%+RQ4M+lmC63s2n$C+QAxKsLGYskJCJ|Z_9!ou>Ppo{07vjqIDeFw4_3sj z7=%T~hGg?Xu3a5nckX9Zx~C@*2Z28(i$Fd2?wuqCE174)})xnQm4`YeL#Z0 zoAk)E4*&PG4EQ|OH(t%KJ$9tt!W$ysd&)rIb81;`Nq87eFIPYR$N6~&n@n;v_aelk zhNzc2nuo2f;4a%Bd1De^{XH%&Xz4+LbxEe=IscwJy;2TH_=VhKGB#1@`s-SL#lf~m zPGwDS9Xo);Rn>y2QGz%VIy(g%6Kn<=Pswa|z2b>RdRU7`7pjNlmixN1zZJu<_6r6! zPM83-7W(+Cp+J&U2k$-_z|qs~N1Q7YO`E?Br|Nbw>8RrB$H9v9B(2-uS3go1;uG37 z{l2*o(ciwxUKh2SD55qIx{#dJEQ~~#jFy%)&nfYH`msBt^RM|r^_yfRGW*KA+CVOIi%>4R z5o{uKJ7G}_@J9b;yKE9hq!Infazy_c;#D;AZpeKObSpYzucSS!Hwvd*oXZ~dsUIXn zmP61ir#^cFh)kv2x1fS8X#nM?1?OI+rR1)?;9}U+WsG8a1W}4&-t|^ZbW*dMNDn9` ztRHk1FKmYGDv`k%WX&hQTiz<6{!rczEewX5{Om#?BQ5_*FXGlxrr2r9Oin-j8Wa{W zQv<2`Md`IcLu%~8hi-7_-HA8?zJz!I`KNVAx3!$PKTM)_$?-J&)B%N`4riX54Cf#h zx*yEaXbR=^G|oWjmwVUYKV&M-lYrF=4@OxN(Pbe?sKzfrxN=s4NCs19- z{81@ExH5b3A9w)#2MLqZte%wKO`eq6VizRY|5KcmSjP)yvN3;D`U3>2_ep0LKW zSlZ1E4J)Faq2KvKX&m*ZN@u~BPcU@zW1hd3`^#5ji+)^FJy{22b5bb5Ie#^9-}x{5 zn?PKV!?A)fSuStqN(p8CN#)-YjK&BphZ9miNM>I9tpvhv!{uIe%-{FLpheU{4{@31WNle-w7ua9mS)LFAm44dKiD828?D(! zYFpo0d(vLsV-|BEJvb;9>BW&1MFxF2d0uR2tdb(sI*)MfKsga?thpx>@Bv;Zo^#>B6G`gmpH$xD- zAM8kJE$GtryIQ|1MP#KY$myzelkaQ??@ zlm&+-Z>X;~vUrvV*!!Z}8NYW6G@z32a3J*9^=DUMt9Ack;5(_*?*ex1WMAIZAaA(#C?nFtNshH&eBgQ;%g@ z99}ccT_mT?8495PANK93x%0qV7kq4W%#Ll_wr$(CZ5tii`Ng(v+eybxruKjDGgW8i zT+Q87_01R0Q)|6~^?5c3MPUknIY7cI8Jh| zWFJ|2M%bB%(X=lQMnK;tamKS*fHqUvafi7b^62^SM%h6$kwFg+JIkRx5Y+I7*-kA{ zxKD=FUrjUcu=Hs)D7}BeSp!$>+N+U;Ck}8u7wy|HAYk?Oy}WRL4iEQmb|h6VQ4j7# z@{v_G_H`A9s)6$k6sq}(LT8vEi=knRKu?pv&SBYQxM4^WdiqMhor)HNIM2$83wjU1 zL&jr@;|O6&jj5C!i=b045u)rh6<>K5CLOSlLwyE`rAbVTVBnb*3c)dgt<^?^rG-)} zA+U%M3}zQ90~qNut3*gA%2mKJ8e4F=(zlY!lwb8*z2VC$ z=7`Kv6E*XTjCsYNN*<;*p~O%{tXu|s2bDWP`(pactT94sS!=HTu$c7J%esT1PX$Tzsf)L08T{(DK)v1*B=FDojy|h2hN&+KbyU9aG zFRuKhUzj2s5**2@Js&{r_A`d|?9C>3V>I7=sXgH}N0G$!`#N=p!h?(TN>`qyW<2Rz z8psIprB!ey6&{&EZa0Bsjnv)_rxgkXietAeA*+v3bSFp1VHoANg{zU~WpqJq?fG6) zz2SDKAg{N=Zch%LwZYu*{sqmI^quHF`Ko!7K=@<%@@Vs!_1|zivai2%l-g;;wA%vd zX*iANCI*g+*(%|e9;vLWb_s>!ERhE$w_gRM+Y5cRC(rB${ zf-|^ysS-Z+q#Pba`FDl3j93*ue~DE6wz}<=TJ-DXf?giV-ARyeN9RW6NJO8KOAfOOHCL-n1fcO#|VY~ z*J;o}4#j<|OmigJLAkG#;HbyCr-VG=v}E8RMrIydgot-ewpk(lOc95=a)zZGLA15bDrLLuq6+k!}7=n&*5ioXIvuJ@7`Z_O4B?8^5R$_ z`K<=Iho;PVR$K_Wv_)CPs;9QiX|TM*I0}OGm7K(S##xpo5m`rW8!BsVzig8toq5ID z@E}zW5ZZr*kD7^3J6RmD1~u+-UdvqzlU0m~1SP|^&@%kGEs3_J18N78MN+;`e$!EI(a+ntMI&i>sZmpXlQw z_>-LSi-T}-y&oj+Sn~F1tsE`Y?KgKZAo_+o`#@$P&v=ffK!-tw{cgjGTNf^$upg*Z zyc%w7U?Jej?uVlO zvpIIuZA5B&&f+*Hqw7I+unnD-QsN@Jh-~iHSSUS|{k~um$4=0t1>AyCJi<0np8w@> zC364~Kw8uC>vG)EEtY72K$MaxCJxI!v$0W{$0Q2XY&r1FeFx(ZBC(Tmfapj+q16 zmn*uObF?e!Ve%;p!$QL0kL)9&antFYspno|Y_Q{CHm{PU3ZRFP==Ufzc|#FzWtm!c zPL}Z2+GXTuuLzF=B@ZgB_~#+ito^3YjBaFdQ(x`FD%V=ru^*c`fw70@O}thYSD{*r zPGtZyR#6PHskgQ_5fA&SVXrvU(~ZsT{5)?1Mcf4wc%UgZIug(v0*lZR4Co|VE1oMB zA&8RSl1WMmEu4okiJfxfe#sa5tol}SzK}?F14o5VZv^gKz%$S$L+3bPK_Mg)xX=o! z3bCzxJJ6f$doitdyQI2@F$~9oS%F2Pkk4}fQJBf>tfFMe{>7}o{I;4v{Y3630%ffF zy37LWtVB}Ibm$p@pa^%gTsMILSrwLlD)bfHZ#3$uX&FsRT_pE`qz@mK@3jL% z`G8M*uR~#%>}Nd9l(~Snue)q2=?{AuoIawzlSHE7=m+~xR;pXJPpaK0-EF1D^2jZrxy=U3@mofru;G)s$2DaeRE+Gg>}#(a3wY z%t~r*;3rI<=S?!2!S$F0jx@8cqx;@XpeK$_`Cyh9F~_!;!fmd*wYd&*G{?^)(RECF z?b~uEwN`VeGAzg`Gl?=&6x5hz_XXBQD%LlT3Ys_=#|ccpqTrk>7db+y1D`R0OpR=i z(QtFur$P~bV7>Jjlt#x-)$R^d8fa@wD*DD&u|(z(9DzYw<4%{GFPL5!66Q>jFWH>S z{ICiTTrepioODal-z<4d2k*C>OII^!VdBz=JBF8>rmqfCZwD3}cUSI0(B(h918o=` ziVntCylmm+p%mlr!UTYQeRB9tHcoTTELTKveQIJVj9Tz0gC8Vf1a01 zcq-w*x>dJVuhlpE=3G%oxl{F_^`4yy@PIF?qSLYEo55gUekmJ)raHSxnS}XNi79-m z?IzT4D~^u;VfP4jAoGlovD)t1tjt*cdCWi6sVPL`H@VjDWQgy!nBw9HYZNR%dsTrI zL{36Z12gS|A$i0#yYgj3yp3uCx|>Z!6;#wX4^41^$MLcft7^ z*iMk&Z8+A~OnX(E<8Oaukb@=?M%Z6m;*ms}9k?nlFV(d(vOQ5y!3GWNAGTWsY5vzHp5vBKxRS-`Pe`Hp^%JeU6Z-^9W3v z?pxgqgC>CB6o{q2NSR;Pk`*L%l+?E}P7@5wW6PXfQOK>r4vl2Utgb6DV+J|V5Vg4O z5lsU6_0Y`3z+AKp(w;rclC)6CVu@Fl!AQdeI}|`XdA}%#t&u4{+I1;)djFX3$6*T; zz|BgU857AmxU7jVK$(V%#NXLypnr4c{Ys8MKra#WQLzq!46l}~ikxN*ME+u|set>t zlgnypwylHWFcohut0+b#nG|w5EaOmPg>V4;yu1-;b|D4_#|N>o4i*xc9uk&HSV@O+ z{Ub8LA^;7@jQkK9JT3jR!T%`Fn&e}bJm$wd^Lw0J-T}gObTB*;favmZ%*8qDF|+5+ z`0HX!VhU3RW5!_2LyBj$BJ!h~O;Yc@oj@dAy}Uj1L8}!0ENb(h0N@~iL=py10H<>U zu2yACB4NgaejVGz$h8ej(DY@2R6A(aYDve?LuU!pI53bPKsBGdYIh4tTKoirnQmYA z)#pCNMrJhyHv4I&@*SzE7|Jms;wh|Y_3-)W3tiiM;cXZ)LHUj_vFa^y0RA5N$Le8@ zx}FReVssoef%2sffkfy=^@Vot?Nw(sk$}2*NS&IAet;n zR+GsSU-j7gp)$D$#n%?qRQkM%nqx2irRLf8J!){Gl6kUL2HybG@3(s|-+T!(*!IB7 zyo)BNOWIy6U`s9`1F@s;{%_h7VpdUiPWb-YrxaXDSn_+n-U| z%sP%;S zA^fKuj_Di4P-pAlQCCZ}r5diz{?OK&%tYH4b)6+BE$GC-sAxak-f$%&BN6B!to>Zf z7EHgr>YXznsb`Hj$ezF>mdVE%L#QkGOQyWK_8T50w0f3AfY!jh8ghu`*@~fV%uc8X z^$l0{(WBg~6Vy^oz4FNV(C*IrthlmEf8|nwA5(P9^&;d^P;E+Sh~E$H8h*~WL$|rK zbc+hrgai#7R!*Y5%Wt3UzE>OlC7&V2bv0ub2Vd^#TP^s7_ry{~F0o&h*fqjo=a><; zWPOktK=X%Mz>3+Vf7O{8!0_)t!-<2vz7&i*F4wZ}T*yJ%oqqvlj8&!jP%bfBrBc-$ z=AVBgKm)&4z72!&vY9KQftR1MjvBH(;y*9FCROfMe_)S_V3=Id?#<9NjF^?U0?9H| zkF(I+$uMyG>_H7$W@5o-Sep{Z?Cx@CD8abgf@(QN0@JY_Y>EXd739*XiEW7B(9#o} z{mgLPZ$Y*U=;mnA+fO_=Gy`AIm{Za~y!a@B7NZ*qSyAt$Y7^tMd0I%toZ^Qzq-hW zSm~KXqc096kfD^=c^Wr3B19?%rQ)ac-{Yp};+w8J3P%+#xagNgL`K$?GK8p7Oi~`# zPN(5yX?b7$RKGIT(WE;~(t=3zC?dTk5tmRH^2I1q)wq*Jp{T___x84X2It*Jb1}tC z&5VCP@gJ(nSA#xQPOkX2Fm+Ya4LA}9b*6RO$iwAG-BP&)fr%KJ)Hr67h(v?8c@kG4 z?tC10^4!BGBGZpok1p(#bOkkZ^T7$ESI;9*7h+%b2ilBvv}ekb)@{R7q6UJ7aq04s z%dESh-v}~~z;-+KSr=r9*Ed5}GHy?SlfpI^AmxDr%a4}&z_6V_Qbq>6X}1j26op%? zfm10qf>ming*>u$2uMQ}H-bE$%Kv<;v+P>jY(Oi*#zs-+sAgyU0xX$>6+>KzvrCC* zLknESf5Uq`f12%};>jsB^4-nYCfyx3m%J=?M}xN(SoqXng?CQ?#?2ZX{^zXY$OCLR|Z)L0Ukj`6x8)2iMJiiy2T=cep~osB%>g01V&RHhcyO-Pox$(k^%lrwOFiN#iK-rZ6USo%tertPnVgM+_Cl7ADV zzjV7xPjXj!E9hE0D^JT`{8J4FXya~*1mMML8x8jZMR{3A?_3{SJxvnvGkNy_gTgfN{9Nw!Xf1cmRp<@R>!E?Q8qs@A9z*g|JdlF`!*6~!p~ zz0{4NpM<7|A^};1h2F12H9ViHiIbHM^9xWK)Re;zN&x4U_p6ux`ad^Ap!kjyx0A(y zji>Yl)}4FxeP0@;J}_6@2JK)z{DMe3Sub?Yih^`CAf|TdMx})(zBAz%I911`U4@WF z&RBX&a4%u!%Jyg1tXhvaaDb8my?0bIgj~ceAtojBH7|daA0vc3yi2kUw zRC?{Um^sRn!bIYc15#ZZ!*(?Vvz{D>(y1ngW2QADs6s zIqBR7WR$HR-1Zz2bEH#!1aG&nqIo|z-0k z%muV?`J?lVQ%gXO{pUCC`xf{(TSR*01W!6N`?Y{w?iC?D{4UH;QnwGkKv2$g z(P+7^_H+q%6cy*NTZtL8H@(TN+ZgStP)LALYPhuCO3L+sA%SXQ+f!4Gj)*)kR(mg! z&RvK(Jds9dn+ENw=%rEEn2}&I0(<$&<12_FfCL$;aL6<9*J3+}qN&U#G+k_XM07`j z?ub2kE{rE7w26eIMg1-AH>qkpeJQz2;{MRQ1vl_afZ4dnUSV!NNDU2mNWSMLEC4Ge zN~xC2x!RqRTxw3FsgZ^_EY)?>@;Nw9mdI>o>dXqoeT#PYJMx^ia%yv@@cWJ5;>+iK z-F&f(cD&(>#GzkQXo23dqrLElaL$=Cq=^82t$6Fu4askhNz2S9Zgb$auVNoLgX9Zw zt}MwiQwWJtur3wcJgIH>u@0{7M%I52^1`U}+4@gSchNs$Gp-s=1GaJ5QC_lOGdpqb)BU zm+z{dLcxB%JMR0f==T@t%(!*LtOo_MjbP>cpV)`i1#}#8r6SO_Q7y+pa5J3!F)gx{ zP&`@!VoC82buD4=n?TOZ0DeB)Acgxs62+ksYG&(>{4}eUAKwxt!AJN}qMIn*BrF?W zQ$2qzAn(OPC9o$=15rSnV@C*}?m5NQ*e1(Kt^*4MmpZ_ym#Z1kTV*#fsIHdScg? z&F^cc#y;MO^E2c3VxU?c;9qUSHJvYuc=KPke2m}$`XrJN{LQvh`at)e{&m0FK~;e- zMn!#($V=@lvab~K8JBKCg;(CoI^|L)2YmXs0y{~lm6LjqdHC7K(oj|+zmRTAdYui{ z5k-nP%ufSJ$rneQ>2&{0YZwOe0JZ31O4G!(gekjTX_u+3S83@0i~0baz5ww^d^@|( zWzlRrjm;1h+lpwfFh?gsZjQOJSJ(mFB13Je>Hj)Ir@FpA_AV>6(oJ8}lCIe|qnm@H zsW!-3h>D3s&_D@%(yee`zq*Me5yD4Nsj)Pzrc6{xSjt%`n)5D#;{|%)0rKwWshUAp zsxN1PA2DOUHH|3r2CzFF%=(PO{9D%aV;)SiiII%zoozVDRvx3+xAtU(!mXPn#Zd;(FR z%32N^j~)~Fr^l7fwk$kgIwhBOg*-S7E#~tqTGg7;ce$HX1W(v0h9D`O=v=a>B@m&- z02-KHgR}~=7))jE0J)y7wPeFf5>z_RI<2wv=y?btUB<3LOO_^)qoR~lzN68@X(C4N z1BAEp_F}tKoc24{*Tek$Wng6E4J=FJ)ETf>ExL1a3X>O>e_nOZ(IAuvU32br(cjOc zTkNZZCsy|2?!W=xPVAYu57;Q!4856lE)C9KkMkJiYtbaLOzB3=gSg`V25S&U2f5wqRjA?Qu#o)25bnVl50!KM*3) zqOy*RDCT)umJ_L+uGV{{B#ZOh3tsYA5p%fEj0}ri=y(Y6{EUr7ZCVtX=E{y^XykcemwOPh zMh-Ezit72$*gV@dD zWNS1Dw5afowS)j`KrkePPDF-<(sE{#OmQHTD13C^6!?8HDU+#!83jv&Jx@qin_%U| zMNU0Iz;qHs8`@(S=iZEKg_u;^rc#DA%!FyGmKx?{8ThWMe0pO9Fl{Tm^8bUbIEh>Z zMq&|W7MQW5TIW7@gME+^U$^Zv7iA7GNtH%&>9`(-Bdh6!+iJ_LZ>AkJ%NEwAqld0( zi1|1pSym|wsju5!R9!>MdteQ8EvXm@^)59N9r6k%s0wO9#Yo0!v|RsLvA+dj$e+A>ZmQ}a)En!#tAU4KsKd3u zHr+T|eov8|2lm{HEuCq!k=<6>O8s_o>|J0}5oUqpGkiap10OQ(sC-kyEVdOIGK`rx z3fckosS&_Chi>fy!&}_MM+t&qT5T|7Axlfis9#J^S1%7>f-LaJH!V>S=~{4Fcu}db zIqheY3zHspuC^##@9z^}?=aCxkgDl&nkjkhb073sXA60m0Zjvk`<9iCY0~lJ3y4A* zP?0^ni+!Utuc;tYV1!s}Z;A9e7vT2$V^^ins%yJr3wE)6g-frE8g#AzNJ z7`kESx*D~xDGQn#a0vF!nRX=PzyIZE`mFbUpQ5wRW&@g@GtW?K0E08VJ4J0xL~;xl zbE1@K#Fj^5G;q3kvXSDmBXr14HwS;%O1HE*;Knzj#gLio=B0C5u7VBmM?t+`1|9Ii zYov2^B_Tdzaf|?*-IaZ$<_a%bEM#gZ&EGQ90pE~=bn5lU8qK+aKUhG5LZiJI@KR5q zaZ^3jSgS=PO9tJho}zgYk#zV1*^=Yy5Q9KbWMu^KGIDVy+K>^$u0sIiTR$H6+!uy| znymFWgIFh#`v?8VJ=&0)0c<@kiBgmc8K$+4Qrv>)!FboVPocR&J1lAa+emB`Cs14( z+K#NqB5b0G<2hTYlOh~2FDRQj+R16yzo60Mk_k<{Ayj3_yQN;d!vEM%UUfznX(;hJ z9^e^%K&K@|j5iijHbB0RYyL`r{uCQbH}gW7A@f*JGbNP6^q)q_vF!wT;9@UGcf(u9Rw`K0ykW#+d6a~?Js=m|dUxq}3#<)bI*;xXzUa76{Ixh>P8j-NWo z#WPi(FH}+aoEE>ni7aCuLS>f;wo(e^&GXIDno=N!lri=kgr<`y_~Z z-^v~3DrjTIht=@pGBAZI?}O4=1vwF}1Mx0@av&~cvDLChOEQ+@R24Ou)R1?oqNUExtL9JXE z!ZKjdra6MAdnz=OxJ;U~k>xnO^0tSYVM0!G%De|x)6%^Yjrvt9%aeJ9w>O%X1RBQ% zom2h8Ap4b#LgZ^5-pvF_=GNr9>P^l;$flwdG{zV9ArRf;NoZ!$6q)_f-5c_JjD9pp zX_2zBZf*CGlb!4_MJRM0+9n{Gz6_-Hn+aPCp|^SjT%^j6M<}=Q^6%!_P_<&PcqyO{UbykY3xK!W`z*8a0!*&VUDM21A7kn}EP%%PcJqvjqEkX{dDWV906}6#O zYTmA6E&RJ(g!-4;{PLFJs2`^)>M6cOMU{$cif+f9wS{|A9lSR%b;*af$M#z_zfLbf zqHZreI{h?#6{C)BX7{=TQ(a_qk<1)Rs13!wJQK&WG@SJ=C>Z9`@x$j31lPj*T#?%IAZ>wY*Yh;021a;S~pk#*U2JH z6o$nR+tEQcau;*Re2kg35(@q~&6Dk#&ao1B+Q}~)>YioWOdSXV{+L(2<&lNI5E#1& z)7;2cRIS|>4jfV;^YH5#8_DJf25sz{^>G{QBnlyou3=hIE6$*`nXVymw{d~p6}Rix z+!i8nAXuv~7Ko)MCzw_qabTO~d3JGu=0`ltFC65T*0u^&FteM<=$jC#$PV-1fm&`{ z1`Ye~C1!BvqJF|cXvB~$nD`eE76^TL`z~h%WFn;^_RC>RzWdebj2^itp)EbYglMDF z{z3BRnB8@LmPTCuJRGm1?5F&n4|{v3FN_>|y@)D1YXvw0GTf{q%24o;ftErLF}SW- zBwZ#jdM&oJrPOrG!`1KR{fp6>s5-ZvSXYi*DFv&j2fx+tJ2zVGr#7gQvP-?|SU?_E zaBLB%E&(iX@75OM#W!HELBsEw@e9}Sfg+asWZv<#0RwezUl5nnWEkY99g3n&1&iyU z(ODiL#|VFQBbrTs(l%cs}Jc)vYrV&o-Y#U_bddcx6af81(1_v+iplI0Rwh;_k?rhuweECGieF_|H zSo2E6ntDGPdxR3euOnH7d^xFDNM$VXG=c1Y6%$N_v>zH&(a8qUYFFs`(WxCU781sw z^5t;ow48$woBkcak73dX!K9M*XG!-%RI(6fs>ad{o8lsMW9YakhIgZ@T{mkgREf$q zz-lg(_Km~ROrDB@V=E!LO{V_Cn`S9CB&vGi#O;S(ESWwD;*ZKRxq(7FHFgFsNH3Xq zCu>)=YLAtGlg#XCqeK#qgO#{_w2oNI2!y0t zfWbDewofW?>dM4|@l}y#r!vitk0iAdtK$zMp=xAYR>9)|hNYWel5FCre!RqjTjlMz z>S0O!*aoob_5wF`4bK!q+_GX53r_-73C4_5bEv3oE?nL{-x>vEdW#kb1~tVLN1Sj< zwiu<_BI!|3gk!G`>kI}M)lq_l3Ae*cEyO^b85=LRh8bCCAsjvjF+VZ*kH32@?sF(BXX+D{M5?U9{XG}1L!CC4Qw>4`(f?u~&R?&zm??@7>hq-*BAVM>)kjtz z@F2G=sMi%#tGM4ekGSBcFnGUnaB;6OJql%B$8LY@9waN_6IkdQ;j7>JmS}8iLR+|5 zM3G6IxeXpP;u4P}pmkV-Lw1G6sTJMathVs}-}qxZxsDI8PijQ6B2*a<7_$&k1psq} zd01ZTTPfMyJIuY|sj?A~!BFWJlm$qJl@ih@ime`*h&q(2jbP1iWDD4Kjdjyz`f`u_ z`c)74W?aC^L!xKLk3|S$ZS{h7e#~o8_Iu^&>594#FcuYdR0!8qTkug#PfZuhyq!y% zK2c2Yf5acv`u-I+-QnqBxf!N7V#9oKamZ zwD|W~dm~a(xvTGSa6rqBbInRq#V&adooN&^fr752Z&I)bb4#SRE$k}&j5kfQI+XZk zUQzxOaRmpm!t~3`<e1YeOwK@L#hSq8`8O(L0?)?ESIeV){ zu|dl~)QqK=FlG2yN-k!7^L6r<&NWJ_^us-U>iS}V&Y*;1cs3kNGd#5B<{v}ei+jG1WQJ};5oEs zOMuy7%iG7x{a${-_&nu-y{uw8fxM6bDkt06zhn|F3V`RJBv;2F5B>Egrsljy9cvRaf<`cYBaO4NTTT_ zQLi2}kvh+1w{h&pQy5cLdb^l9^!D9KYHzn)KEhDBJ~9M%DTI(R$T1G%w1gEij=BeY z?%V_kA=tY$=NB{BW0NK8U?t$AsVDjVF=jRo(w&uLhDP+4yl}4U1&WlpK3}>SvmnkAJ*D zMZ}__f1z5k@Q(r@=j2>5BQS$)!H<}l{tovFhpm7286Pg6?f-Y?!|map`S@o({+W+| z=Hs9F_-8);nU8x?lQWfgQmoCkO8e zb`rpHnK?%eSMrK-49O9W51bYQzVmJ>sQ`Wy9&#&DsmTAi@_+lu<+B|}yLj<<=3F+o zm`i1<*S5`8wMPv|oPNTI7pw|gczjFYpl4zd)#76~uKqJ5+JQ9*?jeY9aHfZL+npOx998 zI#?9%o7{&9OnL{R4Jsw7+r5+BG45JlP{|Sz^I#OGEvw8S3y#{s*ly^j{TjU<=;FVt zYo5=OeRoXV1pjsEF`e$#yJzvpFAHz*MQ^W~ckCgUjv~mklW1$2EpsE6Uf);(JO;+Q zUJ{DJ`y<*TtxkUCZz@e-hzL9RdhhD3dz9;o+4z^2mYMXyDg9iI;IZgMa&d;d& zS(t4YK;2qmptljwG4nTLM6UZWOP~XATKXQ*#NR_-dF_p_;ANB03||CH1Y09u%hjK= zX6G;s=XF?6e+v+JDu6DO>bD}7LnD9vI=3c~NZg0NkD6AQrr0#e>c~(iiDvl>@%N2F z(7o!xI0>`BHzoO@V{FcNyIJz?LP2zhrK96?Jj%$M#)9valuXmpl?pJfCMp&Bso=hnohwBtX9C;qvuL~++ww82K zPocVP6ultOD&cbkkZ`t?_=C-(X1~_~RIov_zoFh#lYK#`RYe zl%2w_bjFtsMlVunBLwE2bu$!`OVOd&TLY=ES(~N)`tT@qhEL}%XFTd;bMQH+4%U#W zMpah12V9w+&tM)R>uUONk#;L>#Yuv5SD)OfPH_T|xfXaGExiq808oVY8oEgxx|#b> zN^97!6cyVE-TG?-X@}e9Zd}HCg!S3CMBhfCvrpkIxr?o|N<~o(a3a1L8-A%vuF3DB<67xw`O|#KoXmO)mW#BCiH=K5;MwML!;=mKq3_Mpbv?jf>Ul zmxTTT-DqEDLVz_yw?!u|snxpr$ zPG`eUYJRFVhIzU%Ma zBJFzC{tv6p#Pq&A>s?ihwDtKKiq+;le#U7qDgb)P`)Q=iklHOYCWl-NX(=~ci(@4E z1`&wic@}waTJcl-l)O%QYzKL5Q;sfTJz6cm7pbc zil1(jZAAzz$z5};h#Ym^9F4Qkb`}@rX3}Z%(R!_SbgNATGGW9nW7+MYk;BSvb@)Qg8P4kL+hTpZpVS`s!HR@B@SilY z(a4KacAD+l2~7i;qXBFT&WrVwVI6MHFvmeTiG?#vqnikCv=!0~W1R&?lbRK%YCfkJl5Lj3!F`r&7TjSuX+SIo#fm zr29I`^;!Jt%_xmvG~WjwiijGfDn$`{HBXWP1AxUvr}s53wEPMaaC+c@(G-v}h9;{0 zl&ui5M<3kvrpCM#D}K7$PnnRuN}umr4DP$j_pTx75fEs-ESm-X)Hs<+x~HufmU{lK zjaZ^#Oh4)(C}gpo^FS%F+kq$ny1=OZfbO4D`)I4KMD#wss{NGS!Gb^lwoQJ;aBu@I zZ(>ZVjIEO*<+&T$beWl8Jp#NkwHGlMil?gG1?q$ipr8I$Vzy7)re5+IK>Iob4cl`P zOO&TF(AFj9omJfX_K)HjBz%`|Ea?LK9ZmTj=XXutY(HtI(=th_*y$k6OxrUbR1cRC zDQ%;F?Kp|m4k}zm)GcfCJrBQ$H_gR9G+*fT*2Q79aMGFHxUY;f=m+M{t#!qyk*>Hp z>qZ; z*b5LiMcKt>olM(?pcUwjhn@F}=D1x25-Vz{-Z`oZuaQe989PvZNZ>hQADYK4g;+2Gg|B8S+xB5-U~Mlt(K-n7+s!hAW5g8gk+Be zM$8lp;DO*M2-LI_C45UV@7lmJMngqug29J>7)aVle3HjCvd?{ha5v=luxKdq1Ze!Q zH0QBTF_3Q_Ka;2i!I?br2g+Jq@*N)l;M3^Bs{D>0^9xxN zj@n_z1wfY%94(<=N?aX|TtVUZSJ2rjaFRP1|LgH5Uviruxr5Qo=RZMwg`L5LnhL|Q z=!6J3nQQiNlL4|31fn`@!vI=9rN7_FB-|4L++@>|C`2(Q!Y|g;6IN|p z_0Ky!BeHd!pxi<#*;<9vjOHon@ODh{ux=ju4s=e~aPr!ea?p+N-?IV`3;LpzVY1TFJ2OZ^OUEc`^Gz(PKAk}EZeBr(D zbwXBC^6;Peu5K5L5oO-{iV@dlixD~UCGiQ8`54@MlKB|ULF3C1*U}yu@*ru5j$OET zbgXb(xP~HZjB^owV+3#FGfte(xiZ4BcqJ_UxirFb!i;g{dengfm5p=d;?RKurHyyw zf@{MJ8|4}~uRk@yTwodH!aLA`6AsSLK>!ubO^J~lU-Ks;9ek6Kg2+sXX*kUOugF7& zR5;q+-eqD&I~WRGi4)GKlC+i0l7H;hg$P0nZ+5(6W?oTUq6(t@4v#ijBh7=;}ICsu8(Ur+ zo1~L#*A6Y-8aAi}uFjT+3;|6p+te>j1a&wKKP6nma;$OJb*b1^I1f$shwe;~dD+8y zyxpV}H^x!?pc=+-o7PSakaZPU#{S~ zxYgD5;W?_L@({-x2{PI%SU5dbpp(x#6E`l?+T{AbC+wy?3;^@N3wHVP98COaWmqjQ z@eE^sqCGJW$cqIbBu0OfZguQ5gRJTuGhAKX?7puG?h$oae=j6b*^>=TCy}w&X|kn! z^st+WppQ{WDfKKQ3}g(K(rR!it8Byi{?#1?8&+7601cr;kByoX)Ii|u_9umL;pm8Z zhg*5ktq^68lWNK7N_W;=0sn*cYj_SMxg0wP-e=u;c4=!E+}XRAf=dNqbekmsp2B|4tfe@0cO!7%B< zLaek$|ECRZ&si$Il8t55%?T>m{VXq(O*DNMw}BXl!U7^=5wGG5{<08cfwa#EKHgKB zn$)qIhwb~q(}HB` zu|~sxxT#>ws&^Au!;o|lq(~P=Zhq+JX+;pJy;eRal2yZhKFt<8Fe4d0O;b41S-NUE zrheEfYa0dsbdF}F2fzj2sqZW%R9qZ27rbrO6sv#~ggBPU4_`iXj$NkN3lZ?bV)hIo;z z7S&XCxn)T@a=}_jtMuseytM;Rs<&%hE=gXcvb6jp)14bhxDt$!g(0Dg%Dngrn zCquT;~ zc`Z=o-`UYYP%Y6KgaLY=59_Ap%l7vYwmQKRI<#W6#*U8aIdC57WGkU39Dk7Wrgk(# zqDzh*KWiThY>2MsiCb67-^{vHZK zlzE5!6k$lAlxaa)xn_0?^68Ue@nwm5N@WiJwqMdoML4J%&<8HP^f z#pskCG?CrX?w1g_ ziQTy)$xF&E54cxTBy9B`TKT|l4$l~HahJxhy>z>(2J)OlK~Ny@;!e!|xcWDmp$3B) zeeXjisf&-ryyU*azh*~>F}z*gd~RUngqsP;<{`B^qNS2pPTzT?Cx`S@R~w`&ut5#= zXH@=V;&e;T8WO&U3s)Cx&&yE?g=RAsGZ{UZ)(bUrcD%TsCjb6^B6Xo$bWI10bIknv zX}rmyx($n7i_3J9pjCWOWc99-v+i5t@goq*_|fjTXRfiEi@Eb=0wrO>uU&PDN6Guv zGQ-j!N1={h&9wm8rVWS0yAa|6B7(Cnp-Z05!?+Vsr$6e1`}0GTu06d5E^mvVm#Kgk zr z^EcTncB&|v>4n*QuDvyuU#zM-wNPvm-KABybPWP`MW$rb^2{~P({CY{4}x`mi?BzT zHeapg9~k{8I5pw7VV%I3zlM*JIEpnowxsr1Ppq^@h&#eSsrx!vM&H}Ls{v3~OR_OE^l@wVKRlvm~>SYX5ac&5V)rN7%0wstIH z@}7sQ&4peAi8*Cq%8`lGoPj{Jd|D=gMATf!Fl*Qv8YX-c#_8hdkZ`o>KU>M%^bQWf zVrup-BTGTkSa)F(ww_)iKX+{cF2Us(YgTbwGLv;)by_l}Y$KmzC9c@)bS0uAbWvtu zDYB$!k|;$Csh%N81a@U)L3}k=$6UZ(yC{58_Iz9_Z}?&qp0y_ls@F$eQaRtS+DZ?_ z7HbAMo%LW5n3nI;(LI~L<8GAy zsw&3s%7OUD*^2Bi)`PFgJ2~nmbo^EKl8>KHb#Q^NXLtMjzMp5cyWMD&DXQvg>%7oC z@V@nNkLPr>TdNZ}!=}n}og}B#ex@zRA5X5w9<4rOh}8wr6t8yVZ#H zh;nu2Y=1*&+FWPZel#XU`{%5cd@ivVmuvund9tA(;ZcYeWtWL|ja*mGWX?n(5t$y# z@&(%jb&&w#bD5Pby=W#+EtO2MHx3H!PZ*p8y(G)bQ_e2@Yzi4Za*Wh1jQmJ znn%r^zO|0>e&FBE86^rE3HIv&Ed9zhb>1R$mPxcU`L&>@Zyoi^stQHg)%kzPZA|s| zW=)ci`I97kS-v$EO_B;27rhoul0N0!N##E*S|!(K9O5~LbW5(P@vGpq?Ba1=ZV*d3 zQa3$Vk~xT*_To$36-O0gXn&@SC!J+r#1wT_fCY2q!T<{u)ANJ^kFF@1ltFHXP$>h_ z6hUskSzzk7s+12a0Te;2{{AK{)bfUAW^N4=*)50pLcN!hb;Me7!$N#*3sbN_QN|e@ zzMp&>3dLuy*)m--Gm6{*I>p+;5*!bJ{l-nvbbP~P!D7{Z!|sG;uAe+7Yd_Xs9%BTR zmRU5r$A;$|fL*SiAHWjV)$3g_Q7WLff0@zaoFn994@!%z^DaT+KA5GR^a!fX^(`D& zYE6D&O%8@^geMwwFzISgG73~FVKM1bWU?A*qa0G~R^IQYg3+6?5GY=syy0k&?I!S4 z%biOwA(!asrNCx#z1={6zDv^^VhYk__*}tla&8G)-62_)V)A(KibINGu8%^K&xegd zvsVGWx*S`Fj4&d@;FHrMRzD&4b6Q}``KH&^j)T_2#nJQuapoU}o3C~|w_JnaaoUG{ z(UAl4tR@LIF&&I-`WnLq-Fs3OJ22+wU`KxMQs!PXb*VoH#Dy+sO8y%y9lh?EWZ=_zk=@uLI&EZ#DImQ_L z8>SW}!X#Fv(ooo>yDSP-%HK9 zD@G)I7j1ur$wv2y9DPk3Zi*AxaH@4^{ONR_z?l05Mkj&zunYNYII#==W(l6^&5Jrv zc(Z)TK85Ujs>NGsOQ5TZ* za^ejhYm9OVcg+xbMQ&e3EqMaKt1d9A;fAa;>yB~_)+tBZmh z#&2JjZC@0vS(fBQ7{*mFjYPED1jHK!MON&^$II}F01wanz@b6XhjJ`3#-mqVcsuS*+Gw7s~dFMswg9gEkflQs>^P=3$SXsO*uPBCAmp^&<)+z2#O!QIXN z3_MYO2eob&&ZWS0wmA}@&9fm#`;bZ7g0(Zz%KYtPFrAq)ILzuWJUPLF-0S|yn7jCZ z7yJ+%(lQbI;8anLa9>IPi+O_|Vng(O3fyN1E9|0#nOI(I`rSpRl_Z)wd{E-a^byWl zkZE-vtdVP;hb!D?4t!-sb7>ZCj9}DWv-9^Av0tYLt6SkT#=OAuKI=rEYk`YV9Z^8w72nki4QfxR4^}h7Z%^jlr9SAEbX?Dra;+|u9Hr8_g#Od zsBd!wqMUDD$UQ(seSx9-Xum6-r=&j>ZOf9~;Aqa^S@yl>wy&i={gJQ!mHr(!EnVYx z>E&nwV;O0ioo*g#uAG=7ywh1OOP-{oE&Vwhl*fm+`FC?zJ+j`RKUODM=GnSf3rc65 z_w(kA}YCo*n(}6#2xxS;Ds`YI@-=iM)mXl9bfnVR;xKOG6eh| zPk(=lm2il8Sa z*Yb5l_=J1`)o-Jr-~xek&R9CJEMwYVo5)@UC3-#EM>8tnL-m`Acp;O;II*B%^UzAj zBJcl(y#h;}V)?sS^_@S`UW53lrn|kLu10WZV4P)dtc|o$w5=8_I^^5Qk?WVE)JM#R z%QlXyI2t_4HjZCEW{tecHIA$NyBE#B%Q8-Qe#}}Zs~oZMy3yQo-^r5eQ#u&@TjA%4=k)sYSJ>;T9HA90(8d;q zEKe4Sy?{(#B%fA%j)4v7bw)R$Z>%+BQuhUcPTgf(o@12juTJ&t$Xzqw;Ypx4nVE7RnNev%mi_bO zTcp{0G$S%M$AfhP+IO3ni<0$*%XItEJ!boh{#a3I)GU}7m-YuD9bT=MZuGrt$g%7W za>TOz-nb)Ky7ecnxZoGtqGNsq(q7gxVAbK5OFGS*&P+yBwA*8CL+HO#d`W9ZFVNNx z*V|U?91qQ}ogO==+V6qy2j`?LD`q&X+S~@$3)vv2gr1U^!(5Be@wH$!{$zFH7QK+{ zSI_Rk_PpvH`&d`W>eWUTzfc_VwW2d!vuW(qvyVjicS>H>Ae_VN0p1VWC#49%im{+Qanz@iX}O%+IJlF(^2eOT6k1aO1gS;k zC9EDqQtPpe0M&Q1mVx%XU`5Z}bk+>HYuWw#Cjhqah zmERj{tdn^)9*YiPmYZ7`iK`WNHiR`sj6wW>__~YBb^AO&E9Em$VSKxahle$#m+<~h zwTpc7DaftIE}zPnu$7*D+4e&F&hcdp-5l?+LWOAV4QLj3X=;^qysz7iOyPTXa zyE$vK?Ku@tiLcqe7rK3Q9+$cQ2Bp`uF||bp5`1OEy^GD?AJxF#n>rm31%tl8ooB3@ zmcH1UO?DJ_7)t$BaSn%kBP~wLmu!=(@cZVurKX+B0b;U|ZGe&r2=ZqP(L>>qWHNG< zf7RXQ^RF4>D0=;#e>HoIoLWsMB1NWOw}J_6`mBQBfs*pidXmLA1nfE7CkS>u&0e^I z_Qz68Q@b-w>KUPWpx1iX-DD4e7f>^?Kbv^M3~uTD1LfwnK!`NcEFxNswzw+4yS6yl zP)Cblg`2O#FO>x>W0s>1fCLSX)7C8^zX~NB)6Ir*%<2YNC$cZ`KC9abkupNhy!3#3 z)p|&az@5Ehi9Oh;r`+wAgerQOWfZ0xF_t~GR;*CKNoy*m@($fFNqSXf_PsuZHgt)eU%4gyXWx~^%UR}XB{4xQMs76*X`P7!ON8C9eG-V`6zmk_9Wv|KO|)Ozwi#81 zU?)|?D{k{!=ZlHfy+WrQF;oO$t~gaz!XOf+;J3Njwpp^j%NibyKyj8@{>qOL=`0z) z)}pdm5tu1vj@zbyV?5p?3e4*iW~3y*zWtrNFpv*m|9u<2xR{*W@ArCrULDN|xLYpx z)n9ps|G7F#>2K!vjkG;}diui<)YH7vQ*jE&J%ZCuu5vxNck^p*uk%Mne|&CY1+BYM6FMQ|@vR1E$oeG4UOS{g6I%v4ZE2l~WDF~Ac#(dT zy5GXkLDih%^I8}wa;a?f)xMvn4`0Zz2y&Z;CcVQjlpzqKd=%Ah_M;)`R-^KAY>$;w8it-J~S9F|lknB~fDa4tNxg(W-_ z6V67nlM+=C*s998h0vU&l7-gob&&Y%t6@9@&}5Yzq-uBL2V2Wq8I$U`<<_^t0)!pF z(wk)sPRR+E@-Z;Sv=@@>?%%gf1}d%oEAJy5WW?^57?XPWTnTpa3PB9E`s~7M1Ce!5 zr4^WP1Wm70-v;w@qq{G^;4FNoi+lP}nOOCpTM$`QoKP~04DjB%U<-gpJ!T}^VfOz@ z>qRb)Uf;3iA)tD>l6hGlCUpc*b($>Nfi6hw72?&0nbJ zef^eCu;2lH$#kY<4jQF%osBQH4J@tp?R8tLSLy;*ztYV$$u)P`!n0G0l^VB>(pj9_ zk&`d(0_;F|yNtSvwa&sj>?S47MwMuL-5AV<%Lk@>{)?^TgY)i)YF%UC60}M5%F0(# zrrd{pl@N7X3pnl8sf?|7b6@GrBy>#|1Y^@SVU~S)$IVjwF7)GM1%oV;7IPNK^AltR zarpaW1#%CY#Zwk00Ew>EuQ=Kr=CrDpcxc+S9;|`;=Vr|bw`gdDMv0a5A7_N|Mquy6 zV-umdX#C3d%5b16c3F3vI_W-{2F=MZ%JoLf zfqFmP%1s3{uMMiXQs`y3Q01_M{%Qce+Um};^`c_aS|oX$Nc}P(5ajWJ-!SO5W{pr5 zs_LMN*OunZYDU3SdbR&>1jIxiCMp^+6cb>#7~E8xy|X@sfVgx~;I>^Az_C$Qq8!V9 z2kP@H7)m<|N1$-ZTDY!=UqbA+JFug3w%6pEFp!5wASrE(7BEgb8}`VlS10ci_c%`$ zGK^see-X`tgSLAasTAtW+z}Xq{J}8zak_`QF(14MUt20zIg9}xi0YZDNEQ0{ISnxA zd3XG|nfK^&dah2Q5x>sHb`9EwRjgrW0+H>Y9aLR4>huZgQ+xTLR#$)Aojbh>wRo zw~c64f`eq&(f}Cd%@gS_sOBV@?;%!<9>Gp-BaX9JnXX0Zj*tROZ8E!J!`9ZIcaq8F zMyc?wLjHlHw!DO@atZ}~#}^Jzln8%jCBo>1m4uf^wWS&So7T3$nI z`~1H17PWqNs3O$FfCg2L71?omCrE5A7(o;Y^Q)Vmmadv}{6QU-hs3xd_R2cFF5+u= zU@rff8MKK0_4qT6_KP@=_IvAF_7=cA`kR2ikM=7t`YWLQ)lWYlHf1NFu%}6!tHAxR zW!Oalvk97lhjb%;W`n@m{o0sT58coW(Jw)b&TDkbn7b+)faHZKuo3NgYxY>KOouzU zJ_d@T{r4(}V8uysIUC;7uQnByK}yf?&MjJE*5<~F6DX9fS$GXdv%(BX$pm;oiFBrv<>(S+gA_$-w`pewIF*uUjOnXL%KU|Zn3ZwY zEM1DWI*Ehfup%2IOwR(+Jh_Z$elrQa>PBlnWP*%L56|mm_MNbLEW(MnEX{a{vn)-- zw=7M>W0oocb0wb1I7cO3%GCV?&A69RT1}@s;pfh zdh}^r-noxdtW2GnvZcVgTMh27<-^Uu^M7P_dbek07A##$QCejX@2m&@4k*(rWydu% zUL=^uOz9bvnT9C6`&r@r zJp>PlT{L`iQuV8Y9^qqzk0h~t3pOnF<-aq7W@GR3t|~rTjC}@MS4Pqr9#iDvh(fRp!&!BrE(Uyggg=K~u1CoezR*Up`y9Rx=`Q#uv z>#}L~Nq8}ELgfRY!SzL!Js)pkznIlRx8qcU2 zf~U)>tde7#b&DU1?||p+-wtsG`_p<&H;V}ct+b6WuV);AYgXokQhmv8KumpIXbK5| zXA#j)v_`iEpkl+1(WL*AeKV3S&6IY@Ys_9~cDx@|TaDZR1n$96uUhoDld6V@a>vmZM-|#AOqljjQ?bfOuEH08#ro>_6kR9d%?`cba6~Sz`cgO z9qlT_AU5|T6ndSc$pcLBDc5U?Jwnl`J(BB#q;ab{EPa82Q@_Uwldk`K265+e1wr?& znRwQ-A>F4*;#ne~n(BBnmM*8QDezJ;9VG>0)Hl95#IY7CAClbNtl0FNyKEmq5 z5~7NS+l}=D{H5Q_Y_1XIB>C#|?KW>vlHyyxcsbL@x#Yt$qr`GvLztz1`(RWP#?|tFZJ^fdFPQB&5iCI+h3aM zA`tWs_nu%-cVgyz&Key(XxoEkh^6Aj+h}wG-FBKAZEKUMRSR*@V!ABuib$Wo(Jf1@ z-VCB$4KZwk{IDut7$}b19>|#~SDM9img!R~lpG=g7kSAliRVSb@V#z!cEnIRwVEJ- zl(yYQT(nke_^CB1@I?=$r~52+!Sfeum5LbDIOfXwIJ7#n1Ifh>F^Iydw;f>Q?@)UJ z+|xJb@&ui5f6iz-h}7}n9>sL@CtXhQYrE`&{Z9Q>sdHbVn;7N*)O`TD=sPuGzX-RN z=&Bt%@bx#|$i`BywKgy z;TIfXWyKC5<;i=4?L~aAH|ZkngBFc0$5A-EkbulF-2 zHGstXaMtObgBoeH{*b!fhla)pe7y5jw=a)AZ)BF~>e(nSilT^NyI=e%aB+Jjcvj?l={f8TW z(>fXruG@MWs}ll78`tI2+c?}uzq?Ez-JfjY_zt4d=%hsZdZ|XIvd}vujGraA=_ea> zO;RtkBWOH~Oc-(TCBbu@H-;@kBrV&Jn;z=M_p+Y^nh$UW4M6*0DHVh0;rGzFsbouH zC*mraRcpbk09%g`w-=h=3OUit?SUQc-DyM~fF3PDf1=ahN-6Lpt@f6!8CTbt+Mqua z98@=iGPnWjxu|@)3dLozZag>#rnEh;y2-`u=5Yy6Jh%P54+?s%Km&uL@|4$=WB$}A z?xCa(2q4~&_MS=B<+8_9tPAy211&{jp3ehzuZ={r4~bqrr@XUmX)`8sJCJA|EwR^D z?l*}kv5f8zn(RtoOsJm;(h&owk&km-G@;7tiy&? z;E04Ygy$Z}aS+Wq%>5Ne=iyJ;z@8pfYN5ME58{vYp=fn0w3d@wZ`j0$#UrcxxVp@b zvbr7O8Rctoqt;aYr_Domn}QwJkE`Lgja=Imrv#H%E3t7h9NYYr*cNb?G1icWwT-DV zWp7NCJ3%J9_5^)VdjivAI-DTO%jjc!?;L0)IWS4YCcNFqauQ*er=$#wnV*(UXh9|} z$7a$NyCm5cNm37s&mg}h95YLtDg@%`$DtH2or94~S%x@6 za*$^g#UF4%^wuyDfuV6hUgO#XVRP-`*;`Ge1Nc<4w9k_JXjTVki6~X z-2}k8T8zlKkyiGhqTm8m&{##jgb^ozu-E zkN=2z+Zt7VI3o!b>@Vo7jRDw<4J@7{apf5a=jdCIshm@zcuLY2n8~^NHAtJcn{C+@ z2}IFcuIBsv?D%>05+BujGb`0veH10iYK)A*zy9)7--vplYQt}1)TyR7A>`u-e90aG zaGc-=)Jg@ETsQ8@gx1q6#O>8i-Tv^_Tn#%oZ+VR;gy)3RX2Y3n z%GDEOsxtoNj?80OsCeHSo6BN!*%8S@(JTjB`?ioabmLb6q0K_k9FKR9Hq1lOB>WTP zhQv;hTI99)J_VX;GHC~z!iz;kDQ<}in|c&9ihX{9RG7;^Hz!yO34G#eF%ELVY3Kx@O#qt80u;l-yT&BQ<T*GMW6*-+n@denKSJXWs@VGqX-}Ko94)dTNbK&|9|(pJzO@KiQ>Aks90T8=F34i! z7~0mVstJ^;`YAngH1S3-(|Jmqg@0{GAyokYZ0f@OB6kfezLMq{At7s0KT}~Assr*H z7?wkd^={Lk4oAm~8XS_`s)LqO&lZNIVMocwj@1 zSdqKhvD;9O6|=u$cFrFc^e5PQ2q;j34^}-zvznO4(uo19H9JXe6zSF_ESyOixY|%s zWC~isGLV`f{!)JPt7RF5W~#9}X+6erYOK;I>Z;_kHE+iW;GL>2xcX2G+h~HQO4Y?+ zGevzr_Y&D<(G=frN3BYOF6xV&um%gz1ztstfLO+Tr$WET#>!?`}WG;zs*iv@7afR5vBPwP{#w zJ7bhzI~dA)7G)W>65z1ZWHr6j(wN3HIulg{nD#mpp?#j>X{D+Ifp|JNUVG0%a{8H!1bN646&A<6o4^r5SpQKVck-xYjb9mdx#8 zcvtvGKi^qChe_|gU$+yGFCmJ$eG9%;<)fJW*Nux8T$7w}Au`nbbik8+z!HxePM_NL z!=HnQE-`#Gz?h5$e@XVk)uDloOwmZbHG8;?J>nR~^u{NDy|c_6uyE_SwZc)1y^|+3 z351-Fxqt`ZjWr-zC~ThRn^BqsP5f^@a;IrdsA+t0c!&6-4j1xL1rN6eu$M;bX};gN z#-0K$4&!c+2XqIkiwf&~eMIP=-`-$FNr-tjoU+`X`(yiaAF;fglF}dWcym=9!|{7i zF7Vr1@re7iK2I57;qZgJIdO4e@B+L!v%JXUsWdCbH6HVba@aan%~~QB8+GV^dJ@qr z&|ngPS#%;>{ojGfM$Jdh-&PyE5#L*d21M?J778^03oPa_ZH=#Y+UXE#ZNsyL}8^k7k zX61%ZDXD2al`67w9==hvJfmPS6SbfgTdg!V<@Ycr>GaYLBXFrEZS|T^6)WKW!K!7~ z_zef7?%G8(KjXwAIWD=sBfiUwkj>oIr59DkrP)#159BUoiO6tw4l}BREnnOsHpz3$ z)Sjo4i*dBcHo>{yWxojPp6fUki_a9Tw+%?>t8d_!br2QggL?8K=^lc5=yHlYu#b=~ zI3*gFu`V@;xRw~sN^YtC>yO_Gv8xOGsU6G&fLVheK1v^U`fHDPDV;i1Zv7Do{W z#0;kPS!EolhkZvumvj3cUR5@(sm}-n*B~>L6KK{1fFKo2{BF+6G&}Tsy5)Uv*oOw9 z_NqZLB|frPOeOg4R2&L#AVeZ+WkBOXPD980ZG*_}x}TN9saF#vavt1X+%i@M2Y(!I ze7a}VmIe)jSXv*N)|j*=Rh|%j^F8i%E!c~1EO2%!KU6U<+CK(>7^xmHyERkh@A@?p zKnZU-_^V;DEUDQzBIa$2n=0KLbCF_QyTrJE&B0i9crr+>?a z4=`I6;FCHa%_H|G+cdtT)1y#V?jS4;d8pKEKb}Xb7rE7*!3Vw81MkKEGs*%C%+_NE z^ldAk%SaRR3~H@NE;&-qh_gK^HpV)`()Un&WbCBX=lt+oyltzOYyXbKE;kWAUwl$k zfb%2K)F{8cYeN86zwD7d%=U9PeKL#hVS*ygh}|BlxC0T`wh+`E3u6W=w^acU-x{sz_j>urOghQ!?>qxhNoFnm5`r zJ11VJ3LK5RLc%tNeFnM-eVd!ba|r&%zcnamjIMzIXo)D(ZNjLox<n30UDEwx=>iu>J%PN~Et0E&hdoh}IN_Z}ia- z4NdtDZ_l}m|C;M$R4cm=icu_s5t&lT)_xX$ksGXdNdR%=`pDJv@X$@=n~)3 z%>rB9#$t%>_Ce{#FC)-_^lK(TQq%OP?#yk)-6)J*NLt~@G%2X;=g^yTb@WDc9oxmt ztgc26y6Kf)LUFqjW6M2Xy~t!Ch#sb6u~4NWm;bbg%HF ziQ%k-G_|5pv`hjroj@8&)!y3)j_?jMTs^C24F-YAUs}K+Behoho1*iRZ^;;JhcR&! zsztWrEw|RjTQNmwcVL~J)9pCa`_oUcmzm*|YP_!&u^+3IB{v0`d2jA0d)=0FD0^E! zzcki#=qFDHsM(d3)+fy6UIu8b!@5H7WF(N4gWCjUU;%=+H|O?BL^o)E*2Z9d2A@9M zqbDkzrB2&*Ija~v2=PI6uAXqfT=kio{$9mdF%;=!-@%B8OtGp9jh8@cuQs6* z3cUbPevxup<=_X{QX?089$_tHx{mcad>8%QrDRw1eXF+ok>Q_@+`Dm6^mX{XSjuR} z>Wu5xwO$NDKr6bf2+O4kBt`*GA5%_rXNTkmv$sF&oa^Dlr6o5AXVO|qdkfp!5 zVEC%L8G`J+`ra<&x*m2ADgGuNU+op6O&~c-6M_TeS)-_xSF!C~XkPfPicU0r+Kr*= zoAKxvLSDbdgF=}<6;beVJhQJH&dKRP*eN=k-qgA-1Z?ne3ORcB(Iy+={Yn6El(;1ilPb4t!Um6r>4V(;8(iJD@XUT>_~?&U(`Zto0YClxtSwt zNqeu|*PK>8z0uUL6SqvB3?1$@9$RyXs(R=y`2`?L-8q1=e!ym;ROA}{RL^SPr9Y}$ z;#*og&OraV4TpM0sy+JU={B8n!ZQ8gi3k3>pGq?}e^A=MIQQln&hZp*(<6(y zhz+FTPP2qDtpo0Jy$ZGsrCPEg+_uMbTn+{K_QFYaDutKpLimAXb$d8#QA=rIQ^Bo5 zvqEF?YUw$9?4pxo4RUZp*&U4oYvL4!vS+pPH;72N)v7@sGdY9=j4eeEC>cGuF{N8& z_-fEr(-U94$5g)7_)vrV_?A;M6~*n%r3m8jvm!6r+|3$R$4=QA!3xAD=TtEs`qd zOKc1tDd$O)aV1FI<1?*wjGQD%O&6HfI3*b%OQj-1r!L;pMvT?P22OpTF=`cnk=W4j zs=E#sfZ<%B4jB8vNY>mtYT`Fk3tlZMYJLi85NDABg|LA+qie$uf~BJ#(0HQu+-V>< zR%cE|6xMqaLlTpd!uhm9TBD;-f+T7nKvR=BcpcQ?8LAU^9R_wqgEqmi_942ULmHz4 z7%=7*v4E-<(v4yYKzc4!fHeF-7OIj0)zWteS44&l)eyH>ITTm<;ndKa;Dc1-goYy_ zE3EH8#V;fiAm!a0RToMuQ}I2oUel3N}!|Z&$$Wqb=Eh zu33S$&=xBY*Q}5lr~z`2CCilH#9x2xG$O%FKST{>3!oyc$e>i{dLtnWt^cMOKh1F7 zl`^y_r>?xvg8mtm5o!#SiJ-t251EdIOvi?&dDH@(;>D6ZRSBf=YYUVvcDywXN-bmg z+2%e<08eLyp_&vV8iit4B9g-RQWDKE3MAurQWAX}K9IEED2}8M&{;y#!C``dy8t38 zY2fnYg;M4{B1H6O#dxdG3Xc>)f#8dtHBJa`#R4@k9Yhstkqt%?_gN5Cs^y34v!ruf zgxUajP5_*yB?n|VFzPH>$3`_X0h*Bjgnx=pp?P9Sn5<=tHf_bLXt)27=&PatSIj1KodnycRVJA;l@Vc z;zDrvK%FQo?Me;85!U?0?iO^Zi4f4ySmI={3;cqLGiV`5!hVj0oGML-{FX>qRDeyR z^Fqidl>~Y@LIPPf1TafhAVtVM$~l~O+K^vS{XS5k00f{fXj<0dLI0r&f6^d-5u=(X3Pge zCJH>-_wUXUm6h@;>+-B|zG)9^|1JjD{pNy1sY-zldnpQvVK;mt#M0-&93J4s-Te_V zR;3BceiCKOAqx+&Ho{dzrjp1P#N^+cVkgHZj@@098P$zw2f=$z>dQN@NV&25xjq>L*2hFyc<$e;UTE z6yq4T?8r;RR#osTqsB66iJ5$#)nkqm+g(iTHcpLW>aK$Ih28G8+qjQ@GkTxYKiXI7 zR2-Se<=CD(oSgSJJ(6w$^`R;~tsyvaVMt5HJyWI;o4iJcRK<_i_*J#FT~hRib$=H1 zyzTzYj~AxJr>(S9icvdydrXm80xXe2u!12Q8O+Wp(PgP9!TW6?*=e*fl@*A=C~zL$ zQM;7nYDTH_kYQrn_|9GeIPHAExpC>NMyNhi<*uqTnDBt8028xRGeZ{Q0%xE-=m}mK zeh)OKX)-z;2!Z@vA$xU%dZ`DPGs}TRyJ2*Ivmd~#GJ_+IY$Vz+UeR)f?pxorMc4r- zExHn;Q9tb|#5TvNAnwDY9v-xkJ2rEO8~U(xAYxS1ckCQgS*bA@9t}~lsqO+(-uAVV z87MKNvqM>uB35o)3&Pm8(zYlKz*%WkbS0P^RzwBYt`#4BGpleD^#qBPUGo54v&O6_C>K(fCAteTdSi3s4sz zfIsOL=7z2|4}}yl;d$Yc8()wb6a&6=DoQ2GOiD@xvD0>584u3gG*!wlUvh5lqEpKJ z=XfNy;m}6v=A;tY4M@@x{6s%NDu-vn(jf@HDjU@!MYH~-GYnCb4b{gqa)=fan&_!! zAHWG*hv02?t{#aq!-#Fez@W9Hl98y&c`kFg|E^B_hNZ9>SRM$k0sqoL(yWzKq!t^p z;>1CCnX={2>}fR-*tV}o6Whsb7`Q=e`|%=OjpMWVn?XT%*`b}$Qo(YG`sdaDopM2@ zI9PEUyf7Yy|JxQ+fe_6VN0FPnQaXEQB3_HHVmeE`CGYc7W6QsSfhnR;re`8$imNg= zq0+g{i(W^m>Jc*Q*^9Boa8goyx^*(4F*Xf;dJ)=38{H@zC1Ng^UWGN%o60(8__F`NvI7R{WlraeiE1li`^pzdn4c#jJYKZdq39xeM~(jL|50fv%Ft z8|f3jyGRp*GkITI1I6<(w3mo1H;Dimc7+&9606YLxK!k6YY>mAkqP<3uTb?EM!+uN}9N!DOym*ciX&$+qI`2-5`tcG{{K zb^5>P@dJ&gvaR>7pj}0Qec<>jW}3&?^GNPqVJ4$~TBXw#Zz;5htZD?(K5X>H8uM^= zW44kI%<^qfi!kGAPmU`9n*45NGG+v_e zh-`br62xZc%;RD{P+*OUN{LQ;2*oF9Nog=Bb7W{=sHvWC7MQ1<05KenY$D3QL=ntC zCg^#p3$TWui2&>){noK@9seEL4B-nL9@g<93=aDm;P46<-7exB1kdw+O!A1Wf43jd z+*2~*f}RkbSFEfk47=&pgk$06MwrO<_k6fjO@@^!m~X2vAs(f~KOpY;u`=sp`9MEK z2{x458)e09NOo!cccH!Qh{u!s`K7eTXcGZsaeR+MLU zfv9ZvxaEv1TCUll0tM+)pFnGxD#dJAv^=5Zlc2|eSV)nHC=#Ci8s!Ee7;B`+xZO}W zM&5(>`+VM5-tdnuq5PM~%<#V8`C~=b`-|XC;;1Df$H(AV<2hZk<70ryXEeq1X8p^d z9jfTsa6T)YPZIFw>Z1yu!{HscrJjYtdjqgYf!SQOZ#H>M6w&vZc+EOrvJzk}l9Fx* zk2!XzSxb{e3232;Fn(KPJ56$VVU=t)6@`b^=f&d7KQ5jD(JI*8kmbnOiFi@{`a(btEJVeVMvN+1#;6?f{P-ot)_rqUt{swB7ow>1 zfx#P>tODy++Z>_g8qi){uj2@HXZln_-Qh%2LV%0GDLC3yM=aw>MNZgay|6#f zV^+FGRx+CItR&nJqzA7g%T`dT1cD);J_iN>7ma9TI+|%|3>g~t6?%kAj?x6;Mj;fw zx%-(|uU#sX*K5%|TV?@zbg_5lG7nf#jD8e?$KD6V*5hc`4YuqE3FIQ>5?r>sdC4+C z(>atSzoprckM-rec*}})wRuUNdtQs;k6;yxo>H^Sx@tO1=8I<%igPwVJwdTkE#n-^ z7BrU?&+>e3s`OPdT`meS&qQ*w;u-THJV(qlBqw=Mkt2l&ml17AB|FcZeuR+F(3cm6 z#qhuyEY&(*@}HbzfQkuNTD!sD zmphBmBMmnI15JI57f=QYs}N(5iL6RST`IXHI2gQ22%7of?8E!>_g}vxuN3z*DIPFP zlIr=mh%Ta1(gYRJEh|B)FlB4mYQ+=IilhlCVMWDDW)0;j5`9PevZ62uFcxgY9RNKj zYCsbxcUWym&T>&AYF@*vU_`==GIU;HluRP>9BcDdg4{5+MT;aZa06M8*IC848SLQm z7tdVRf#*z;ov2HaiDIqvainL8ca$~g_DIG$NhdQ!DI85cQQeXMNuF;U5hzyR#j~GH zyu|6Qjh#5XvwlvkpEuUex%IQOe=kq}aylGh5x7@tR=om=?_5tg;*S4K+!3*=Sb+ta zb-8xXE8BD1kt2t9_p@J#9OY%+wJEKookUPqo964B@CO5oUA&+qMTV1zu zo4f@j_^n8yPSO#Yw@dW0vYvVY;tm5eEet;uV!NctjY|MjCwK+LDv;9{N_HVqa(;0v zm5gp@%#ihmi&umd=~nQLJRxGKM5PhAsQ@sWQOZ6~sPymkRo9oARLJ#rZ4ny1%1=DoEX_m~V`-2hkO@qF|?&EHWcZ zn=&5D>k8KNAU|?f$)bl~n8qDZE#0u)M0WO`-x=A_cL8ZmB*yUXp5{a}k1}GU zgmyuzjRB&3bp}$FaT9m-O0;kL>}exuSj3@IqluL?GeESkq(mqIj_yMgHd6sfJn-UN zNu#W&c;Zp{$Rtok%wrNLL!v2p2-JQ^05uR8v}DbMTQ zuC6XV3(qT$jB5gPOyo>$tTIdoR}KIatWr2mjK_-JFiEy0OIQj>QBaW-JgN{~jS;(x zCg$pT*8M|8*Vm4w!Od#FD>wAIAxj?%Lkko`>%6DKI9=^>sq3^CEMZd85-cpNkaf8i z_VKMLrFBz?JBjg}j%IbQ5N(C0ugShKK@b`^>wHBDL_+J18+O7q>Oz#RB?aHJCT6_~ zU=Y?>J4^o{3RcBr)r4>j%!A%io`Ha-(|$a;{dWc^qa6a&Qh}gmmtQeOZn=k*E8>Gpm(YoL| zr689xB3%ZSd`31ilFrE8j5uWdz@DLyVv6}i$1>R)xVHt}q?AgKdYcLA5=8CT-Ab5r zYZI8Q7|MX=U|5QHTJf#^m#N6I$kV585oKV)N^^E{dV>jlaLkieA3RhS^e1nQ(E}rN z8NdGbx$1t8toKE%Q&`_SP}lSCc1X_)>v(?s&aK-8^}4Y-o#S#oGDTW(CsKGrtbS>- zbY&G{9Hc}8OLLgG37J~)vCp{s`guRIu zq=WX0&CV<|=uS!7CezZ>j2Co2 zdl6V&n{{Qt5$rJs+9YHv(#60TvrSxg0J+mjcEL7VKx;L;8v|9+ltOC_2pk~D2@N(G zb+$>osX4}H=uJ8#v1ZyB1g{R;S40DNKj7edEXdx`?Kg=7$?4Yd4boC~hhqVOgJ8h% zK)}%;z@H@m@WJ4}T}|OF@;dzGl655unv1#st3V?1ZSAdYW1(JS@$1q0Mn^n+n$v== zSySpj&RYWgG^8JRFOOmM)0zBS&*TRqjP*_>FCK6z4Q&ReL(>D7MC81xbQuHMvIJi= z4Z@*7im{TI^&J9`2wG|Mj*KU~%67JY12o_}?yZpzV7QJVB4|Hbw4>L^W9&X*UWpF} z_<3H)zzcHYNLCx0K~Gx99g^}jbNX%{1iM4R!onbk&_Oc_UE301B%)`<$o+mOU&!j};%)XG9A<^pVF)ps_x)bP z0)}0rY@3N4)9knjNT{Xr&I74d+g0X`Z*I*B_2FJ<2vi$h)!8Mh$RE0g#nHMtLb8bf z5tS`~c0wkoly$C8BZg-js)` z56aKNsI%X9t-cl9)M>tzz8V0;?Y05u0mAQSveS%?3qHmhJvY9UkR>Y~RL$zmQokqx zolDp5rur>FDDqeXY1qIPW6FfCFKD%iHoQV)H4e_;GwL`rYT)uB5suqf1sH|8<|x*v zo28p2$1lB=c%n3VoMb7sbPY!@danHsNy@OEIFkS=mFwdjs0OyxEtvW&CMFZ&Dca>L z5YP|>mAgO@Mp}xxNHrdNQ*H2FZ@}-ZT7nDU1>Qy*WB-y>RNaT?i|5ZCy!@j;{(xWg z%;ZL#Gw#--Pxa`CoZy?sl9WPJrw`s< zJ?%I5+cJ9UP~J~*jBRKQymRiYk7o`3$~N?tNg{d9Gg<~M=zny%4_Tk6(XYbzlU>{m@ zu4NsXtF`kOLq$h_zvZf@}km}ofN%qG&n#_{PPXl%@7(DqwW90 zI;8NvYnL0r_s{01{}?(oArT3CjS}wO=yU+9*aOqk)0qMWaC1T4|rv>dJY=g4fGq=;|!jH*D9` zy@UAyPJ#)vY0s;!2;`nuCG>ha*6oJvFd3rKhBYn`RoCr=Xbz9bfPHh%WhMWI3EIe) zyuiWfYGk7A)!C@74i1>jS%$+pbdk*fqgJ}ffUrsiP}Tpj)RDcSkQ=_$>5~2lm@r;5 z0*v6r^?|v)`BzP|E{DfoIos7AV?AenOEbmPDxq|_G@&Irk2F5A6grYdQ6FcE2`w7Z z)1Cy0BIK5q6;JAnmPFmtwJ4o-(Qp=2b4)K;B8oIl4ZP}q)i36-WhED>2^e>u{O(0L z(fI26%!-wvJI$qz@sd^DpdYB)oSs^gyt%Op^4Fbl)p_!ZNT(dS9;3cCS!vZsL^Ngf(+NT((;O`F zPj&t%DcKtC7~Y-(Z&*Yvks&dv5?hgp_3m=3l(Sc&kW~p3V1(SX73r~TS|Spf!juYZ z>rgo=ByzMivA1-|vPJwDA~}==hmPiyWU6iV0kP6ZQf)@d?9rG!OxH@g z_S>ERqlD^j3lpd3dF66LOBd-6670?-qE@uC3C5HHDmpLHMg~>KMJ_nSSbbj1$a_(p7cuBVmJI${F!^3o@S8_nIvniCh0_w5 z4qsR78Y)_0!ReU8iqC+&a$XQo>J@5*78xwQYaSyi7owPheB+UdNYs(FSAKs;O^3R( zpgkTR{1&QCY%`ib2~1c#s#a0yFgzhSE7y!{!M+e|GlKr%Grlf$#Ns~|`qMmAfyo|L z5F{#Q^uWYGBJjtoj*M}QUgv#|ii1HOl+&%cS^uvwn*Ar)QtmEgg3lR|yjW+f^PRSR z+g+&EJ34yE222aWFwjzss?YzGDkvn==tZb*dRsiYVY{Q*-t9a(FOFu7@2qp%44?Hj z3(lM)`0nUXaU$Y`MZ7W|9j4u$Z#;ZwDFws0>fe2@q)@KJVuQhwwt3ui|@DF ziiLo>x=kBv=M%lxrKsV)nG(UHmcrSr2~e_UiR&h5-WV`fqFnMc#fmpD0AJ?NQdAq# z+jws4Fl~>aMiV;cC83Q)!q0-Cq&CL@oF#uPYu1hgy+SDNUu67vU}gH>^oiG z7&7`;N>0oV%f{Yz>dd1Hu=WT>6J~_2OeHU=y1%?(cTqIz_#M97OHoMKx@1_)2(A}g zx7@?6myFZ}yJc$IQ!aJYVG}cPtxoQDg`^diD{!Fx&)4F;sEj|vSx5#_6nbgGTcxW* zrbEhC+6$`=${k~NDOb1oY}GX-n5&@i6dMKJ@VRv+mNYY28*biowfAl_a{bTNg3uX& z?~Vdv=W8fBjur)V4S!;#(6v0RMtd)M+La_@Pz{3b$^Os5kL7xzn8nAQ!~`zxp5fnatH9>#6xx^ZBexNUe-rBJ~*uHj7;X z$w`FA-JquY;(wTLq0WCrM4?-FegacCC3j#s%;^nUrO%S8rkNV-$rtnI?q~#2bep;5 zLNQ7+)05|?i_X-ch+#WLXS-(Voms=EX|aPY-Sohag0o_GM|VMjVC0rHzhz{@*Be&a z(YD|ZhkMLTel2$9reM5Ukn`&JSb{+vwI-d2U_5MXNyq1Kwx4Nti(A6d8HBDNcgv^o z#fyV|8hLYrBgN9@eanMR2dI=GD9-t%hH(6_Vf`}NeQKo3K>7j+h~-Z}xwu2A+Ih|( z^V)EG5@7e>my+b;QI}qPJOWQuqtNIOOD#5H)U2~jb1if!O^}j`w2c@(fGT&^r?(F5 zUp397BK>P{vXy%);&(FQ!R>-6io&8V3vz}BUxJ5G&BLpTc=p-3XND%YY5pAuuI4?G z3&^Zj&1gb&p|}b7&Xqpf9KUn-Mm1Nq;MvRsTxrh(2&?D z!pvM#+1;jR`hviVA=fR8^M^itY|L35xe27`uRKiD%ghIu_KTUd3ciu*_x~8h<6BH8(K#{zvfq=|N!R~-7 zs>Si%r-4P&NwGFFohi8i>bdKTVK>o}YGaKi_r=KnLx&>v^bB_c%Pc5Ggxbt$4w!-G#__ z5*Mj;^nUL~Qp)^S=O(x;==e>L1P70zTESIxV#tmJ<@mC2aGzTMTie zC3{KcHnL2QH{9t* zH%|m#M#E@rb^|sFl5f5RS2z!2Zg21)vKz9l-Dr?{ObI*1p4I9d|rAin@!s(!I?H z%>)=C6+=|NVky4pHAJ>C{~c9Q$!ye$H=1?MsoNUSy&z^GB96YhNle8%RTqI_wr2Vizz6R9~I5RRS!b+LxeZFfN+_1)gGc zxy+VQl?2RFR{Bzyfq`l&2*Qb(;)49pwm=w7HW)gN9sGy;;;3ysTGq)8tBe;3!Nj+9 zMV2(9MWQZBKgy;W#vE1;bdD{;s4y>^P-7{BF0B|;4j8=&^C@=Jn&??3#Y~YK)9}7A z4!Xdf-C7MOE1D@XR_RoI%k!o8y~p@z;ohGL!ZWQ)O^_;krF45X4vabI)%>a)gz2{Msr>bS#A!!&wKW% zIAi+0+ScV<5Nk&DJ#~9M6y4`5K<5!m7RSep^$M?JKR!~4Lmj=)+46}Ie@Vr%Oe1C+ zwH(yXO0@bnhnBmRIrn}HG-x-b9hfH3-hd>|$g*}*w>7_2j4;Z;hsMwi%eJIsY3&Sb z)Cw1H`!m}bfFO~YGLoatMjj!olKYZ(m^NPleKX06?3R@~vK3MS=>QT>rQ~bK^a{8q zX@wdj%+4rVM_SuD0xOQ_kmtv=2UtdYzuhXN_>(W5K7aarPr-LKYAgO1@Ly|{H~&9h z4cJLMkqvrIotqMAxcB-LUVt6_i__MwPri8i;_34fHRl(nYW9|8{>A+HDe-`Vc3RXk zW3pqLKQBz;vM5s>L#{W`*{qrw7V=PhJvaOouAg;d?*xz=a4GPLJjq4Lp12Qv(~i@2 z?lOG`k#OB{OLJom9Q<45Wk>UU(mwFkp|S9O^r$Ycc=Nw6-h4QJ_vZc8+1r=o%Ko%1 zfg&4x6<$~|=0kk#VR$D7uwtO$eXNuHJ@v=9V)3*qwQhr_VW@(#aqzJFtthJoYKF-L zxvaP9ccC;qCFw%ll4~MLtZQ;TzEXy2!0{*5;;=U0Kii4OO=wokdbDq20GA@G%?em; zLATBTGyru`ludS5PaT=-^-p|zz}l$K4p{(G`zd1fZ}h7eD5O2|cn~17=CVni*p}j! zr%siI7pCaOpOp;!A20u`R$(wv6^y)ZS+TA*FUj*mCb(fVW&a<0Z~xu6ZRL&c&;2WK zrr&cs(^|4;nl{~>-S#-PC-u{L@mfx@dy;MjA|VMiNw5THkNV{K-@ga<0wgGrASlW6 z(vUwS9+3or3*g?1`}XNdBN-C0*sG{+=6RaXD~Lrjhd9_tRN78y%|k@o6Bq>oeR*?z zuJ;Xqb=gv*K1`6#csmPp88F*d3lbx`Wo)GyFarzB1o3maEzKb_7IhpfZa`1kq{KqZ>6|BAtqBDmH)=-)*ch73&Bki$qW? zWJgYkS{0&PuE_J=viP4uE!G{Xu&Q7ng4a#Z0cHdZ+r$(=8v|9HwJ7kt&kJWjB{I0g zyc(XlB{i9 z{Yr6f-^1iwuZ(*n`L=^>3*@%_GVN2AX9tEU8bG%&3AH0+QU7WPkxXGysb3=brBRS1 z+C>VX^XE-CeAc%n%mn>M6 zNv*^K5eXLcE7e3SCPKG&V*W7~W!4$Ph|YAIx(?wF1(Ug{BMuN&vx~1uO5Xz&AjR!$ z_wN8|(vnF9c}DIRs3ed&LS;>IMbo@H4%)pEv>2*Yv%nB$>35_8E-eZD-uA zSP&SL>833<-yT?o5KXE|;p=GAu>t<+Kgs=z>F@r1dfEl?_er|VJzK65eybCD%OS@u z@LP`uy|oAIRt2!t((HZf(8FN2+E@P&$Su#j`k}Tq=C1{3%Vd;yVQsZ^+-(I2-DtKG zRjUuR%OYy!6?q*rtz%QlyeZ%r!N36Ye%dsuQg*woGf4J%Z?aX zUIf2g@Ud`$x|i-Q*jWAeSNr9!crdRz@ve4EO2Ogx={CreLbV#(XxmW^SEnddUaE%f z3FZfAU|M}{3r)18#cfTz8aloFO{ZQqg>BjYf{+qgB@WGNLzy{(LEtsQ_mE4>G41jFK4I_G7f+7KllKd$|CgzSd@?0h zX`b+iE3K(y3sw}kTIS5W;BAvHT=nEp7KQiwXj#_Eb1cY%)|=VAmWZql@tLE8jL}dPVGus=QT2*lrp^xymBd@GLSHQl^lCu_R&_l#1!%F@ z2ba}D-X`a3`*qEMoka*WP0g*7nBAYqm_9#-F5pV(@tHQpNhNvm{K=HetbTOtj=Or! z+u}kLRY#yM!s!$8R20yoh-M^V^q$!)AkApjSatbXXDTfTtXf5aDp!$RFnf>e8x|w0 zC{3=?LKG1*o*#U~OC8CSGU={OJIM!R@Fds|f~{X&gwx5s0SDNO|D$B&>cTSkoE4Hw zrCCPI<(-#MR_JO4(rO_oewA&%Ci*&0VaY2FF`eS3H12?`%aiBPD5>D4u(Y{t&z5^#X&E#dh`R zgC}q)a>1}pQT&yKsOepSuv}XkdyT+{hPzfchU$?(TYWYKp4qGStXQ$7FxeM7m7+cMmws7nIj;ACZ*Q*l zlGVOAwQX6gpKg`aaAU@bdmgc#1B0teDI=Yu4A{fd=`aC`&IIurQj zf;f5yjjO;I{K%Q%MbNgMV_$W;Ko$XfgXMfx92EecW{u zM=W10!k`EAqc3Moi|IMbSLEfa(y$#bqR);1-Ii!yJ(48h3+4gv4uV@18TWzzZjI*1 zzM*IL#?A$K?fgnfB$5ORm3UXLSRf&)CR>hfrg)p2-u)2{Z|0bGf)j9Zb9r|0`f{4a zzg~%li4v1ivI$o{$ryl57#P|Zg6Nlf+qu+M`qS9@n77LyH0OVHV{OY2%M^-=uZ@PLtZR z#_xTn$J_+V-?23TXwZcy^x7<%Yl*1{h;UL$R!s5Jp9*pN1RZqthIN!a>L%?CTeMNu z{9O5i{R{^+ux47VRR>y0jR>zFpSGcCZy_mRpK<{GqlC~8lr+6rOuOYU>Af1h==IZj z0SZ?{6;{_PXe=N~sel>xODShL(>UeXC&LZ;bxfCc-5HWW8;gWJv5py%r@?A21H7*FM?UOYroDMreR#?i!V`efa&v z#BxCB2j-PD1#84&r!0|8*UAg=cUS^`N3zZKw1*XGlx~X=qJ&{UJ8~_#qZI(l+v}E&DGo2 zqv`YrS%@fzRJ1UAUjOdX0Rjy68&$4zy~I$o+u~}pkxN1ir9NIsr@JNy8dTeZwRzA& zOlgl;Zt2)jYNYbp9i=i}EwWi*)>^_`VeAUrfMyRWlJR;3!Hm}s@r-%LA#`mrIkVwi zh05|eNEHldafn3j4|B%rz;YR{9i=l~p)wk;FiDM9pN9{x#vZe7J{-PbsmK`@7107H zq6A4n?YB&1N?seLjGK$jrASJJjfX6T*G-;YYke#M*gj;|)nmZL1DdVjvXCkFv3xVU zP)2vCVqp^)Yk+IG_oN--8tyZBe7fe2gPS^nW;&#jP=jXn1)KD+dzDR4^}>tYg;@Fm z7SeEYO(c1kB|&oX^t?*AvjId`nPSC)M(mkuVPN@+rL6Ga!`om;oAb7!S!__5BUS(- zVeb_y^jw_Ju1H+)dsayD^lM-0x~a9&>G=;bPr#SL1C`-%%bGK1%hlb+QQ{E$j;NJ| zWvbU3BxnLFT#K?WJ}+BvDH4S~b9W`e))!R$Sn4B4;b3+n65ifny1Ru&*DjatXk_o0 zOxYC+XN9^AxVUS;uAE9^O$1kYAD|On&a~kX3$&xE>O&*#Dl|1v5Sr(S*ZCeq>rBM# zx|}Cmu4aRc{%tQ~@KtqW@57-cmA9eA_Agl`Ip`%D6?gPB^wDQ?!wAUfOPHzCqi?;b z`~O{kil^2U^&8)2x%&=@Xp+dydKi}N`*n2PBphMqcBOH>2rWe*bOW`DDI$Qg;Z77u zQh~{*84;On;F!=PvEQld1^9bgFtY)C%WibGeD=cP<;G-0TtPHTvidY;0l(968%8z(QSZ+-=5KD+Qnr z;E$fVEzn^Rl~9Wg_;C{#6HN>KGvi&l0SOY&&t28pSUhA8w205HJM+hyDp~!!tCZTG ziT<6=MX78KgU8{u?H&7=Ua&R;o2fj%>B;ggiB>O{&wNK@j}+U%sfLy^Z>ytIZFH)A z9!|9@bgF4dns5n4bsGG}mZ9=n>4y3(E^1DP>g{IQm!paIX)?lW+Iki@Dud_3fR!J} z0|IUJ9AH|;d{n29AbvhN2OSS&E_%#*c!9D*vrM35l*b6lFOTsQuh%PRmN|`>w;ES{ zRf09Zyc|v_0@aX04Xh&}p;t78Oh@7jHMTv=k9?!$&^5wZ*Wm5)sYBc#s?LKsb=C$!8Zo3i#h~V0{fzDVQ z!l?Vxx?Zo|YQ~covR8|S`|#*!Iz7U`kNUt+s>S4wgk_P4L%r0xkA+mAR@J5`*Lj-S z{tj|WV*d_`%4F*lz1&eLs#T$?;*l9}RU|Q3C$cR_lJJy6@qic@50grgr}*PE%F7U* zVV+J|DvI@S)r@|<(XM&oSqQ%@7>UZFV3|tRkEnB1lN#Y#X=8N%e6?*wZ3tXu7d$Y9(LUWwTu>~{ML~-- ziH&J&J)r)DQvhS{qJPl8m>2tGF{Ui$Wjdgc20ZUla~j>@ zDXAhS0;@QAwojfIKX}S$pDhqg(GUnBVy5YB8^S&{%)jVD4cMzsO(V~=p7}k$+7tvCrS^C!OE_$p5upC-ZBZ=1ygzrXM zz`B}*GBbf_A7uM{lDf{08cx4(vk`AW((Ot0cvHMNm7$%WxX$6s*M!E1QEEjVABbeNbWOhE<+95b~h{*B-00k z3cUfXNCndbhB`KdNQum7E>{BJB_o>Ch$|0{V?m=k{VMqEpnJz8dS=3#-Dzv4_^4GO z%4N6yMyt1@gi1@@)^^P)S(132(f-t3jH{De81E1CgHUo^m0m`F(|JO9N}fu_$faG< zcxUEUm(dzoG-Jg?1s|1REii{E_xapS0e($^)=oaFmR0NR?&VNG9G1lvU|B4jxQ?^y zE9su*anG0n)!xA;B3rukw(1?K#X&}RrqAJVNfQ(V*!j*T&u3TIEMKnF4rs+N{r$dPKesH+6RN^vIi7s)Sj`>ptKQQzizhVE z%kKKS^S(%dYNs|rv2HC}%_`I*t*8y!qXZWKOXn<(YwQ^`r95P;u)E^=yYpkhr)+Bf z1Z~kfW9}e2VpWbPSQ|Jc*AVSO= zZd`W--uCWV#Iu}5Q-ja*4iCnJWvb|@uyrkB2fQ5nT(G~}=n0sz%5ED`#j@UjWt$C~ z31x7s7vP@KG8EK)I7i?nHp}xi6#!yYspj?GUhv5PFKRUw$zo`N=SKm$tFCTa++1;($Fv4}}Yi}f*JnsH@gqA{Y@-&Z{Y zDa{Z)tYDGIq$S1FHBFEE0mdx}dD1C($N)b@$`p;MqJtu{U(*UX z%S5~Y!JJ5#u$;~6Bp`C`LNo)vSXmlDEhRN2Z$zM{6$7$CRR)wbtWj2d4Vfj2NnC<9 zUNh4^pJ1?MR%>^ID0*8b{zr9`EzqJ(*X(y*>IPY+c)@I41HHfvq00;xc9Z1&hH1^S zmrThzI1}ZWW{gIAF_s$;b}$XStx{vGXg?U>BuZiAl9s-GgCF9eLp-?GA`bKCZ9uox zFW1QznA{`5yoC=0xgAnyMQvi-k`=w*-O^lZuagRC2D&tX&rP0(xI_XX3hgz}(-xAi zYA?;BX2EWujmkgOyN1=9xQ!|@C;*s=?S5l$h#p?eJ^WX|?J$nF3ZlJW`b)u0!bqn86G*eQ3DQ$5Uc{4}7HW+)n&YbX z!411@(K&a3WkOpwNaFgIFT^H;R`fJC8 zcUx%Pi;_%a6O*i8Id6>KZ6k=$ra-eW&CV!o&|A0pv+FCvq@QLYL$-&{hAD1nNozga zU^A8WVe`pDC9G4H$*M;@i?uueu?U6?Sw;~H+)Bo5=prH4BEXO*r7Bu6Nu&r`?u776 zDw-rUR<~II0yIwJOL7&_Fc$7)-x~{Po={_=T9jFYp^jXwhXmWu3&*>bRwxxVmLhD6f+_;a zl9?>CC)>5vz;(Y#NN087)HpNGl(5RsDZ%_b3BQUC-Ju7jVJN8>z8m6cX-&&L#?#4rg703^v)ZYyb;O-Du8B9 z$uJuz**c|B;GGY2JI6T#0r}Mf!49i7x9in{+^|U8vjP{u<@=mwl8fvmYV$S#%IDFje; z&G>|YUdW1}o9Rvs0Zi>L-KGf)%?ozVMHxm1lg3ag)G(wnXZ00dR>-b4R2f2Via7i< zj3LETm9klDC4zv41G>1Yxql3e6O%DQ#ntcf}vzT$vfHxb^WPeHD<~4M2;uQp6|S`;IopE_)wDN~fbZD4r=s)rg;lLYCRaSS z@V&6l&1xXu(S#3KMeYdkuzQsqlQ%-?|6M``$-s4~uM5HC8=>I+4>CzO{5__~`1d&Y z)iBEoDs2qxN8Ohyma>|BF3D9!L}94A!xk`|#O_)5LhTJ7*kVin%!G@gF@p!|YUKm( zN%h&jupO|ULIg>b76M>?RIz11pjGoqS+Ruba8VHaRecSVQF!gRu7`!++iQq<5wCs# ztme~p(S&QrLl3FLM-Gb6D@b(uFelp`=5$+g!w#d?knF#X zd?~oaCs)~%y8dT_2UR*8#56pCdx7PPiOJO3RDp-7wvTyk+BXE%*lEubitd!*w*Ji) zT7a^G=rnWMshwpeb9T!J9n|EWhH5Ce5tf8Ch#_o3itIuB&ZeA<-QWZk2zOgClP$H; zsFTPXxGl|`ZG!-qKxe-&yZBpmvQbc*qA&^5Tj9xo|5c;3^yBf>R~@ENFqsPs(QPHd z01ajZkk&SAPm#PVav=@w)8?KDATnAdo_8(w2i-JZsiyGY%#wJY{^# zqMZ;yt=mviJhoL>H`rB7EI&HJxEo_r4_&GHD5cyOFeRD;fpqwOWNUj#h9KMyUV0@< zh82@>FQ~^g7iDJFy{+=KKSlNEZB2oyduBJ(iGg9ATxdF(iUIay;P@{WtSE5q=IeIO z74(@{Fq~=l(lO3mdkp80>^KvlOUX6EZ7UU3>=^M77m0;U2^K=Bi21@q5Y8ND=rI

MRTzeW2zYa>$;*?06ms_*U;Di_RhzO+ePDX?}dk=LI9@ zE1G32aSAb`Ypk=+R4E#*P!>?4`BuWzkt8c26-jBd;#pN9GpovJ_vBCHqEl4~F6!n|$i^~L(kryXKs}NC-+%um9Bl2Vey3|TA4HMF4?ONK@+Qes-Ow}N zNCHf|gOFhT#>Wo+X|0 ztgi1^Pe7j+Nbf6a=$A}k(J(0bAk>OTo|cKC855;U*52jxAt#|8$Nz8JV_0;nc8F^W+QzXdVv^O|jA0Mju$Rdh0B!+XsU5O-?guPb< z^?p1OX{r_3s=WoFnV)q99UTs(48)(YGJp{GJsHXy9tQhLlM0QW!%mnsp&eoNo4EY2 zGg&4%V26dmswx3U-c>11V8&)=$!4JoG*I#d$^3zf&E7S#d98h}bpZaYpJEMnnn5}~ zHgF7y$oRKxK_$o0_&aMx>V*ji-18whb>{?}`5z(dtp9Gjd?te0x4GfW+b*JDWSIVL5K|J-L48>I5|8ZUNn7d|%p z`vMB~hQS7zXE!Vw=0~GsIhKlUYet6!!|YijO>HSV`vGHI_i(r3p4&uDQj}TNJ!wIz z8C6s=)n|@t>Sy&kntQl=#M}b^6N?G$VHu|OESdbtsCikVm4zrO!dSQ=KmWDiweW3C z$T?A`sVzSm|p`=F70;0K0{%5v!bpTzbB^6GGetWCFTGcK;^&kZXal` z-A#|1=LL&AQA5Lc*V|f@<0(&Qkzhfvsw$d& z&TkSIMb!}qYCC;Go{9o`6w!<%%oY|i9A=HDf}S(mhhlr=;O6f<9;CV);q#72b9AyYUrIql{c5sB6{rPD4Y`Y05&Nh943t3|d>52%S&$?@-9a2$pA{=BLx6Ddc1vO^>W1me^oPZ=*&w)tzzio0HDQ}ees5|y?rn>%yAwRc&gQhoz?)b5_XBt1fL<7gFVlxeq$#4& ztg4b{VFfNL&avMZ`Z;Uw)c12&L;+fKZrKg~Y9HS?p;kYelyraS0vv0AUkyTzHV|9V z)`K!4?WF3|$<5{2#p}yy8t=oRSQh7kXZ{im8#vnb6}xuq4-S%-slVjcnJPLek&8qu z+cB625aQPuj69(eP2v5IHEK2FM59)gAoa*9ZCzz?hxZc?MqzcFXHu>c=X ztsE4toE0g@+@FTF=szQMYLbsf;C+?IYKwKNncXP-y=+i(q-)ROZqBWJy2%9df=w#) znFUW6RPVO3w%d1`)utY6nX`60d^=fhgz`DMSZA|fLg&)dtWA=24FqVcBXo(#Fop4y z!J5}Fa3H44sEUNl~(Ai$Y}^vlI8Q1$aR|V?5+keS_o4lkr92vuHS-De}2KHbl7VJ z?9KhvAS*K%l=VLQi)9lg*uy={KY0S9>(SUh^2h^@_IvPLE+fgq7%n{8;IVius!WgcgY`y8^U?5Vk@kS7?3GH2| zbsOh@IlCsb)YMZ3rg2ZrpdRB4bDRf{(7dcJO2Q+cZ5J z?8ox^uRcI1{+^NscJ#lB?}}$!ajHZy$Zybit1_XKQi&9I7??46Z{3PFV9dK(Y&Tfv zg~AyKXnvLHQ%PUp=4Sl!G0E72=@Yl_lm9SgML_#V{`6Hae0dF3Xc1qHttb*%#-cvZ z+^{M6M>yN#|Hw8hr8w08Drth`dvgdgUz7ady=%N?J6cb}o625N%ARO1nwgQQf@V_B zThm{1WSpEw(YUpfUv5=En4RJYJS}QU&B4} zic1L1>BFOkhq*Mcu%NkJ>q95cf1+_*e1$T2VKk|tH(}gDG0?2?Vn0>u++ZEuo?k;N zlKk-xr=ZFG?!_NYy_5f8^G*g^l0K++ac^~Icx-Vah8Sv7Xf11o;kBBVnoSOiuZJPT zwbTgL)5h4?&pbBPrnezu3SM~bOOWop&EzoC1ON7KtwH?F;bGoZ55wOMa?t1h=<4c3 z3%kyHE|GUN6!HY{OlbkvA`A4Nu*cN3Pp~vgxVY_vn-@Iw|l-_D-k%bmNe!i2~qo#q=?4 zbDjZ{kK~ThBS&iVd5(rVqWTq!y@Tt5DB7>M6hksFkMMb`v*`VU%%{-oUo(CtWk~I{ z%~khNSJqwz--#K=u53PUD@7E>;aXocC4_7hrJDbFww1|XvjvUXY9rY=8k#z2THWjW zaP@{Z%S=J}mA#lJv@k3>w0dPT`M85zeFu_lQW7ey)HUI2ZkF<6aOS;fO&*PJS8V(H z-YiisQCiP^hdJ~0^NecE?93TfLWyjL%$l|rn@;;{c;dXjU|YQ+Fn+mmB>~Q`k%=O+ zPNk+xX^%bnOG>Lhq)C6@t(wOJ64~^_mgZL%M#Hq3Xwob)60pk1_DXwni5_2F?7T5W zp&mprq)12em5m4*b8j<=wEHyjY3;ok?7a=Vy{%cAO_k&b^w~a0MX>GmT zrDRW4hDvODN3&B^hP}@pxa0@$9Q%SXWax>jISvJ*m9v)~n~T`q?953D@pmW{8pcHI z(`dX$kg6onvm!DV2Fnyy=nl6>h^Cb02m?=I@sK^xBKCkq2ci5r|Ps#tBz4nFl69bnkv0@eW2qwgBSRRxAGkg1{u9Ir+dK)&C z==B<@$T=k!PWYW`>!WJbi?9Uck{1i7v@MsGGuVP7e7Uj!04#zPa4En(Jr{3F5*Hezv=#kKjP}X-Rtp&0Yf>{ToOMwBVLeChrcfDI)$P5jA#Jxm7SO6Oo$LKX zGeS!xCQydn23t?bj~Riw%EwUmIHZud(V({bl%hDMy@YkLku{m#99?oEW_l+u$tg{0oH2pl#oj)6jb!nEt?6FRbg-lCYj6=5&6pr z;1!9~V2WfhftSdN(b%F-YqcoPSv)1=TyMq&kEjX)Gt_U#EH1TPZE{3cjEEv;SRQ{t zBBwzkg95h~NCK{VIOnRM#Tu0tJ9nB{AwEGMg`ro7)(F{Ges!M=2DF29HXDF1xxt_C zRZ#J^HOoh-?4A{C({@n1!vPL=#oru) zz!}^m@xWKMU~mp-cQ91Z4j5;Tfy5bz#W|?OJ7yS_4~=tZi_qXuJkDJaId{P1d?ZxP z1L1P+0gksDHs_>`G(68oM)Z6POwVvs z&mm13{Lcg6dhW0<2EVW$x@TD2I9{!1Ub_f)vYHVz9hoZC=2Y2nGGotimCy9@&5lC3Grgxg zq4UI^^C8p;ZG4b?Y=f)XJ{H<ZL*j{u6vsoGTt5e&O6z(9)5zd4NbQT9|l zS)YnE&M{w44`V#LHHA{J%E`hM+BbldEgcVR-KKk%Sq)O!u=5^%zQUDKZ-DOR*Mnpr zI>=wse4tCIde6miAapKDWwT@PegL_k0=g2``*vXC^njyDGnExW51O$r$Ar;nrB$#A zC^>;^N-LaJv%JuQt$byha%TY5h_hIofC=MwsbZ|82{H<85p>MVi2$}XIJ4L6&oKT_ zJ#HJJI^j}x_l~XQ_9#N-i5+X5I2or$;C^TZ(c9wK2%H(AdJSX)e(sqIDs##0n@5t5)F-IlO29j+T4vVt*cJ zd+xG3n{k@mG{lm#(?LBA#0GBd_7TLWG(d-{sFgtmp^o?|On@_p0K<%0{s5{fNNP}< zK><~RTC9SvdIGF=uq1Z`V>tm=gBo>$v#u7vOj?WJ4riOzT@N^6*3td8jF ziP{>}tS`cABZ_NKyPk-z5HN6ig@%A_4J#-lZ1V~~hvC(X4t45|7LAkHAdj#!C2$(e z9p*VDuZ_SPxOk}6fu%@FRh2;9S#sh91G5H1tYmGQtflGTVH=ScE`-gi!v(}pi~A63 zd*E>huN^Ikq1N>s&49f{BmzhGflE!`HMHPFV6!1*De(Gmrbhd!g*~at%r!LmzyXSa zXdVrRAFFO-`MNOSKg{ekXyNZ;4Y$GdXV+H)tqIPX9rgt0iO<%eaD^5HCxD4u)WiC2 zi24(JqTl`SwAHnjT2$6?9KmA4olqrA%0exCYjDtTr;y1X21fIU2Qp~Cr`XZv)#UCs zR{9yMDq3U0dySW{k9~CJFE6M}IIrjHO4X{N5d8&KC`^D@qy$|xg>2Ef+Q z(R6w=WOgd-0&{FD)nf8T!m>!jHV0_Ii-DcgN_uSra4lNECM#Rd@EwEW8#2J1M61on zez~{8T&u}d)&GDhTbow}Nn-TnI%7QHDL2_*nw>H^(9iG`8<<9Ud2AY+rYsf3`nYOP z-yA5|>fV^vAlisTWl^w9CF>8YnOF1+4MmB+O?M*qsP zdvZ?;Z~dmWYiH#;=J#9<&y1ki`t6{tL9oW=PK>?z_D&(z&8Q%wBB0$}lqxTkkyV4I z(y%&EITNqPL_EXyAwRt-zI2evu}s(r?RYB*KH8>7_5D77(4E#w-b)W#^a14OCe zp{*rWEpTuKpQ{d3`&WunlX`(g>@12znWP+F|P=3pmWXS@x zBAV4@lMCliYz~KebQ#ql;3yAVG9p)^Okz^7zZ-vaD8t9`-#Wc%L=6Fnt04^QD9yRS zo_P#D#op(i0C^F<0v1 zFl9aGgywl6XtZL|>G|=1*)}EYtKzmvx#Srh3ZP}JB_?Z92t`%!3fdl`yQ=Ib^+y6* zM3qeyYk+4FraKNDxU&($O-mw)Ewtw!PxbjXGP_;N7yuhnf+XQ_16n#Q++H<4@!|`z z5@jK$7sGr?N|5L9CYqG&`C5^LQK`uDQM^pn(sp(E~eP3ei=Tg($s62^wR8r|k*ZdK93dwf@lpB7u3^J5$h3?-V^6 zz(pH2g`JTQ(0{Q5f}gj=1HlFtY{{i6hEN1wXf94uDoGS1EfdA_q@s>@5I43lBPCt$ z2z5;F6p>{R(lw{t&wQU&ePEdaoB@cSXE`lsMgB;nDV5LA2}5YqBZUk$nwEDQ zu`l~BPQ@~sd3~o%6kT|nh=&ZJ+89~#dzNjiN)li)`kF>}EQ`sv(h?_LXEZgyw|OD% zxn7(s-e|igQWj0qSeMzBPDizBUrLp70AQqkxR3uxH2oL+K4KxdsBUxLLEL+xo3@5w zr_U`(kC#M6(tDdCVe{Y%CO0=>E|GjqTRye89_2d*y5~4@0s!D&}#MhuzRv zNb)t5D!X8yw7asllP;SOzS~f}tpKC!Jx%k3$?4r6!PaK>)JZXqPi`*HE?!?w(|8|{ z$$=nRM}7pm2Dr0d)@f3#MM)kg)>$;jcegzMid<&01c)7~Ha^p z@&lmZ?F%hi%>CzVG@-7<1BnHBfRKZGeRcz{zO>d9XEGOs@RK?cXx*w?pWVn&t52AUP9m1CPE6~&Gou5OClwi2q6mj0I$t>72jRnEdG<~_=p}nMvJ-gPK+19pc&b z75N66+8?L95DL}L3z1Up4OG~~&~3bE2jvRbkilG6?GV$@LKIGk-%_7j&3W=8LPKw< zM4t~d^l+7_)|ec-oJ}#n$Q?aNA=3*&r;21Cl4X*E6VQoeF%Ujn*7*UbR{1Tnuf&pD zATYgrkg$|{>(P#S&}Xj)Nts+`be{M~*N^cxH|nlo~{;?do;<2(_C z{w;HU3O3>lcrR$3nPFWIF&sGWSczp7vzeh1+@n&8i1z{k!&(3BAaVwCfll=D~hL} zYP$O)$w5-tjRL8=kM3FUZ4@5%i3^S@?R_+d~_S+EftD;lYXtUG3oozT#tDmi+ZG5ENPSnwNdI;a?A)`q@o)<6vWqR>sqqe3s z#sTvHHBHPz5Hf)KR{QVl`ievagUn%?NCbu6sApry?QPX)i&m$jO0hKW!NDD!c%Qu! z?;s9=RhiOkqIWR3@f@k@rNE{TnG&;CQM7{P4^rC7jQ^u#Aw|+?EtL=GD}nZQpbu&X z8@oOdc@TAsU^RAxidJ@mJL-pP-09;SsAyk9_GaBevsx7%)BwU=_IA@1utQowq2b+PGzcC{`1z?=Qk8aGHVNcwKQ z`3Ph3N^Vcl!;bKPaH?^==H-_k{_}MavCu zo82~{2b!FVRI}rDIJU-gJUO>9C=&Dd@GYyC!(nYgSB%Ca5liy)`?FWiTDpqjahB0M zUr))88TtO~RiiyFO(0n8c+f!!xGqG>)QXjDV)IY`NnT7}oK8=5n>W__UB5!^Ure7r zpQ6>B&c!{OZnHVxq3?msdnj}P)-2fwldZ}VfudFS=7G&gSCuBeq|#>{oQn2E*xIGP z+}>W3oE9`?iWTSv@O7cT8_*R-W9Cb;YxL!nLBBL_;YK#bbsEPgf}o=Xlex&Guh>r5 zVbeHsXDliUuGYv52e|Y=4@9U}-Oj`|G%;)yhtltv+87o2Hj_+gK{`)pcIOx4f#*W~ zK%{O9nn|rW*q-h%*cr4mshX-rZHC)-j@_EX&Eb;kgi*;zO7FC$)5E)+r0-HP^yXoe z4XtrQ?F=sMOrSprEI>Vt&I|D%VHpaN2pYSa)4GW8bWd}wEYbd;5KPxc%@ky?md6-( z2u;dzj{lNy00A=1`rHxLv=Os=mgqG`rEZkO7BPwdXp?MzHVhuT);3|y)#uFHjFLS}rics!!FkI4Hu zmX+RsQZ4tN_*L==c_U_eUXVRKI+Mi4T z7FgZg7lCRKra-{nIC)_Y{!{WsC{xPB+~E?;pFS0!G8%wD4H=*>Z4KBaK`}7$Uj*j4Bb*j zZQ*`L0f_~Z*^x3aL15P8StQc#Wm$i=oo@YcC@IhMxV|E%9+Gy4rsQF=)Y;uc^1P;&%7&9S5qvDo<@)l>G%m6)l#`?`eJ9fXCZ9(h*_j4jSkf zR$2ek_GUgxt^BY$`2#2&ULU^&z_J;vR4;9;bRRIBU25Tn*TFwj99eK_+Fk4Dhd}Uc z=!Fo~;AbJ94wGmGe^o343Y$$yC=NA50zo-)+tMBL)&#(7WIO-bY-bByBVatgc%Bo` ze8lq=D`q8E{!~GCGgw=6ePqH`h=gW3$C zUZY+<3dp`jN4JSRji2Q}-gR4ac_Wpg{sK(it_23R=iT2m35 zLb|n2_Y~y3>HvnpaVDqaB_zOt>rhdiK-rG^InnR+4A7s?8zb*t_W|oYDvNt|T^D={ zCF5n1=)WUuz<3zkz-|IT3FPwp;xCuJ(b6oID|*L{^$U5`5h9A16;t<|f*)!gP&_Sn z9bJM5tiN@2@$ru)cWgcDk-a@~*INIzm^-Z1zgOHolBnH5%my<4e)0M#i`GZT8VaOa zn6Ml!C_xPNuZ9pI8Hy;?Cmj8PF-R!xA`nyh-b`K~BY17R$c~YzE_()EU(HnO%1M4n zcl&Dg{C9m0bMG=qc{XqW=q^6s0AB1h02EEAJ)94>2%d{f6(ZTF(1^MSazGQ;3!?qV zH=I;RWx7*r7#5F7gTo)mjfZ`PVDW%f-(v3yt+oWNOR9bSiD(BNkq>g|(F(pye`edGn`AJH3!&+Rag3n7O z3W-rtCz3BGw1`$*u}GB#JE1wBL?XM_TC~XIG>v~Pa~4f#7EiF}O=;7uW>Bm3-7tua zZP{f_~!#B^^|e=5Y)lxLb&Yl9HnmP&x;37aFu8`1tT%M-EI%XoXS z`eR{yfiAi$yyJwhrA&NDeBn%q-1>yeSh_R4uphgFYQKp%SCjVF0&0Fqd%qHJcm+P{L-&`Y97w z*3GKWlJCw^DQ&kq^;Nrebkh?L*lDlCN|}@%%&Nwq|I*B;Hm};U!eG6F!>sHz{#Oh5 zNFtF`Ma!-c_m0GJzi(J(ihc(T>h0ERo2z)5GDTym=yX{<(> zhWxHNv-78w$MDsZ7Y+HC~kutsd(jX1mT^?53Y(BAX~yq&$PvXHe2MwXK%X zwJJB{sai2!kc7@zB1uYBWNNa(u}T!r4ymEpMg#OS`>3rh>2z&}0e9;{p8Y9Tw1|=E zwCPD(UDj)vtf*dd2tW-0-`LEQT+(P|e?_d0wmwrL%QU4$ZR*o+n;kwt6YW#*0?r6+ zqa1uv=oEaLCy4Fy0@AHt6n0i_3=xHU48C<4eu!SfC%S7fG&+4{Q}K(&g4$VTmf2xe zj?+%?es)IxxCF+tNxatpdziFGQ?`o;mF5+_w}o&d2VAF|MUOdrzech(`>jtEzME$G zw%%)5UhU5{XXxW;I9-DYS8||*KJhYxhwNcGt*u+hy0&jQIZyETAtiUvzTa=h`zuI1 zj>)KZr++v_tbMmz<9++C-*w>ad%SOVPmDso@nh>|@88P)P29SDx4S(7!u+c9E!-Jj zfjJ(3z{)%BFXg6)g=zTY@KJw5&M_rJ&g{{jDh z`r-@x|LOBDzWCzw-=2T*hcADB`o)*O|MK+TPG6k<{`AX#Bd3QMqW>zTqQ$?Rp0a{R zx#u~l=wjiykApd_{{NX=zpi{K1$~&JKc$qcFm++TwNkH%3D#E=MeA?(r_(QAOiwMy zZrd}HsEAJtq%F^u(?}FdNd1+c+E(7LCX>mpYPbJagyx*RS1f~+g9^m=`LFJH7Jo&~ zOQ}S9V>k(9v4YX&SJr^3*$K@ufq^uJ)kNsnOjt2lvTUk#hIz@8m=(~t{q+6m^!e|m znEGpnGp#$_coqrqD-(}FHiy9#t zrxtoQr72^t)uO+)+DZr1pDLs4js$Anl&t+I(C=GhX_$Sf$vafK>VF#Q!$W=?xpY@{`6 zXvvk`q_tCaM@MX>ARM(QDO;yySn!@1&}$w7F)Z)_78|GtZ3EA4E@!uu<*H4YM7leU znR*H|!Jv4g0az(S3T?71&IQku6E3lN9x3N3R}z#hCZX>4l$-%^(3F-FP1vw0aGCDr!`tBmhWT~O7Y19|6eF>ybo0gr#?6GTrHy)cO})L_ zG6&4J?+juJ80mhv_D$V`R(y~!U$tgafWU(5GO-DGdJRCa@?=ezUZ!x+srXpCX>p6U z?c#w3E^yx+71y+KnTJWt@I1yOS&0WC7K&wtzuJgaJ<3k&3oTeZ&Q6>Isx*iCiu~`> z)1RL{{o!=-?|=L4v-Pt-SARb^f5*<>E9dXj`CGcb%V&Rn_W$i_pG$bk!$2or=_e6c zqqgVjX_J5v(^%Uy>yA*844?q1Bi-Pgh-P|{<1%7X^D!=+lukg2@kK_0JJng^P($)} z&tZLGoCCo?hd=!%Ii0@v!f(^`+53Evs|}hpLRR%4VKsGqIOW+ZmM!(s?)k3snQJ1^ zv{4HNJjh-kzD4ZMhgy;k&tvBbN`Lm~VE(-PY11+*= zf8Kdh0Rsu#RLD51?W2j-+!&n+TRlU1ZK600Y+p`U;PCvEX!_7Vm8q3ZwMAFf(5U;cIV!vuct z^+Z3=9pN1o<_`lO5ZUc=7Zf2WKb-C!jSbdRBJ{3f;xvO$HQHLtvm1(6*-oa763XY&JXG zo8`sc=Pe#__oCVQtrx_`23v%|(@l0HrREg>^~y#qn}mC-F8N8(@Q3hBO++!l&pW2r z_G{4=FVNY?Go!*35UL}2%nHDKhJ%*K5PN71hW4J>o-?gGGF2A(RE#lDT#`g&%buLu zkEb8v6*#+hjO9NeWquk)r3VS`%W7FsNuq>NZ=FICrlH~WxlAdB@q8+*_Z0>e8Lle|D69;n=nuj-QK0Q8 zTp=LYoL!V4qEvaQ5G-?05?#tv!6SrW=#{rU%^p@7hooa^R^E9#U>f{6m>&`?VW{K2 z_3NQt`w!!t{Mi1P$G$fGe=;g)&QO0pjQ6j{|M$h|7hj(KVblM2`sL{V`ykh+}3;7L2DPwWx^icnnA-uQ zZnJt(yC@3 zb_E^*CQPJvSlgU7Ikbi#}59! z=D&Uo)cr}uKSlN7qj~iv$~6|NP|9_BGZ)X1Am z+{GS!$XOE{PUqc?F?vmk9`=hmXkh-^Tf?bsox-aNW2Lp6u&%4&>-<)wX`IKM)9A1_ z{^E4raiGBO8cXLWaJ>y0ONgJlr3H9(|L1e#-tUX;LqxZVdtWlM^~6M zJNRm0(4-+{znmwYdbL;AgvN2fq&!CJ1L(p$lZw$8JYp8m-k$ji2Ch!8|EuhRD+@~v zUldHPu7-r6-ZbB-f&8T95c+l%+S)B^6nR+js1f}Lk#``fY1}9l1G?Ym7^-TO4v0G* zCIl-J7uc+g^A%!DrB~@SJrmYW>9(?a1)FkS^)XN@fdUuKYoL~#B@c8hD~9x#QKgDy zbhrwIZ+#jnl;5@7oRY5N=Wr+UTpt8>G)KlKs zhu>A4w9TGY7e4mb&TLa{hO>5cVziuns#@b{IcxO%v#^|%k`>*nZDZfthYzstZBygE zcdv2ZkNf^Z?0Zjhk;r73gqo1gn@_jcknxERXhd#y$>~v7PrTkzybfzdMiTF5M@|y) zFjKS(f_p2BJrNH?>gO#P3MR`$N%B-WV|#dbm}2kr*l5mA7K|!=%%oxt3qu?1L_;K4e?;Gd%}0FZM@>skGPk-7=<14YDhWdsdo)b3 zoe;%>Bi%#L#6DH|kJlqWo(!O0C`Q=h0G|XypM;}>ga+v7^2f@V?s?miKW$8-Ma#iZT%IjA^%)z} z*9jjS!ruJ`s5WvO$_!PLWw<9q4(b}(p1c(Xm$(k=E76?v6 zL=)TU*^*_-X?tg1Hp}QdVSl;3y?zlis_*T>66RF`?|Q68odW~k{b8&|3y~z^;f7)4 zV_4;$a~rEsZ5BO}I;REO#2Ca#~QV6;@y*5mhAea1|rT$VDCN$O8urd$<0&FIj!AkT8XQYdlt|uhjUy4v`^>Bnec;x4QMoDdc1ar97z75XGGgrFp0VPePRV zddZ_6A2ck-Wge)@lL7iX3DoIHfL>3+b$hb2J%K<47GDO2y*8Y28%BdRR`aQ`l0(=B z$gVa5GP0*%7<<~XF$9fkMdRqgix6`i*AM z&Tl=E8^-#MuW3Pj^{RI%G#NPU8|8 zzP)+n!S?lYbcCD z=ze`$gr|=ly~9^-UTw|uw>PiAu*3{baL+_*f`Ap%s-OqdQ6Qg_&=zXS=>U38bG%jtTH zlT=fn*;jQa!VvQAbpyK(6$Z_m^eb@*?yB>*Z{A#<-&Q`v>4wLrv&mgNK?nAH0H1wX zi1cz`If5sSOW69Vp|Oo1{J!-zU%T}T$UQAMohM9^r&^!;>H6mFfB(-h=yo&v51Pf~ z&0GAJC)r_$TlO($%(st#VjjI{!+VO4Yn{l|6;bg8+C!yjJ139ik*sSCXpa3VJ z5Jf?*u1#=?pPSOGl5cK&!ef@1FfRMB0Gu-ibr4?zXNuXvrh**qTc|JR+qYK>B2umt zi;rP9RW0iGvFt+U+dT*1;4A{)!M6)%z$1_qUCuBbrW>hcyT%Bvp8494(VW8+T_P#| zF0>4WI?~+(25;69p+ycD)Z{BqLfg__5f?v zA25k`zS=vV>Ox8+dXK|a(W@Cb@5*^4q46u7K}cKQJJ)S3OkND8aw-E;_^@Ibxn~7m ztf96fFjK#WqHfIVi5ps3UvYyvFbp_xj9uQ&@OP z5B6y(6P~w|$N3op-1sd)mB4 z_0GLMGu{;R37$z7l?9vK@%%fm%LOZY7hFQNFfyu!(g*vySEmc3{T*8m436IBuQlu) z@UjI@!c@8d9z2FWkKxbz(QgcY?)=sxg+HU);mwdbG~XI$6#6cJ+?D@!fW~%+`ehIu zdczh&N>-emzaqEhz0P_Sdfj)sx>gKYsW3&ii&u>3Lu5oB8ceSdedw@pe?6!gT^`pR z(KKSdg$#v|08^c#;*o!=Vk zP=Bg*s1H@A&-?vPzr^0o-5-n7eca-7hb*ktRHVA~rVRxP6sd-KxSu{X)@bvMc8ro3 z-H^X9H{=*4;a+2uL|13WD2b6hjZqSCeP+CAtkLFNqlN9$Wpks2?U!X?8*8+6^?R(* z*7Dd`qYbamrbe6S&a2syv0(1gVildvDN_ZHB$CzJ0iG$g?8vYl%dm}Q*v2w!V;Q!w z4BHTTjb+%JYb?XoIgHP~4BHL911f}mNsqW-lY-G0NWwFzXcqbF=X}c5$%YrvX>L23 z=LuA?u}2upgMp=Dk4k%aBgcw6qqA|e=Kpf6`J=PZy+&taS7%3OtH_o{XQ)C{X~(O4nP^iv>CV=em7oGsJQe{;YeB`e4z>71sJ{ z3)1LTQU4Xd$9xqEnlT>$IHwnn*-2(O3C*+?qDZIYZ3Z`IH{W%i_RfyU+1c6IF?s#p zS|NBd^U~g>ac7ft;LP_cz%>GO^v&$p6I$|O`o+I{B1`y=hbypfr^_T`9>wGvVCc%bnRBAGT}2-U8|~1eY|kWQ{lI-&R(}R>XsjZo`7Xy zMei|gjzurNx&Zpj*_9oQ-ry;h`kc|uLLB{MgJg%*8CdjUzY1AV#H9Rd_kp*mH`u53 ztU+DJ?`P!f`pS1k9cwst4&{?NT#q#zJ4WzX)o>&R^f2rkaP$%%wk7|7{W$^HHHM-a zx7kTCkXQ4ySwK<26)QNMlCyit6R```P9B3(GLiuCH!zGeu1Q<2zu7aMWyojs?@*sq)e7L>{Umx_Wf*fO-L>a9@~@s*uFC=zyd^CnDF{eDJn zM8e3~&6{IFGeR{p%&6k`488YyX*=p`PdPa+&b3KY4C+{w*0`xcOK&RUs6}@UWz?ca zE&9M(^sW{kD6V-f(p-zeR_3^srk9ZgO(Z*Rkrj}zNXjfS4YHUNOy(kM1zV`#6%4|? zF*s~y&$Z-}{rmJ(Kh@FzM?kp0W_OmJDoW;Va!S5m>BhA32!b4bC7NR&TZe`r(L|4pDo^2!ybyYOla$^uNiso}B`s*C zn4ZtMD6wFukp={7&WZw9X5j}r**J>n9&<^g5(SH=uz+M0{gjF_Q!GY(q_fp}bxo^S zP^me!a|7A8>@1csTu$D83hZSZ_Y1$GUuD~%MU1Wo;ba6*HoJ?=9}|t-MiM=*x46D z(vD5ClP~75Vs07vr5xnb>*2P}8E&g;Q+*@EGGXLA5oPT9h1%t#Rk%>#R~C*rh8%tK z_U$*XE`K_^zJ7Iees+8H_RUY{HS+G;H{=@sdoO_$J3usS+=Ez6cX zV@O0*DpBa3pSWo|*>a-91lpM72~`VGq+KLCBRNq1j!H+|(| z4dY3%i_E$?D-d~RR#eRcTU%@hK>!mYxroUsh*fAo=xRDJcvP&3f=E^-(BzI zwN4bs_>QtqN?<^dv$jo-PW}V`nCPSBpHBY6p~gh7EB*6Si^)H=I_3j(IMrR9sm&Mg zsICAqTW5;CN1?@~ZIMgUmAU62X$*(v6^yi~YQ_=CqNa&{Yp3W_RH@AVC@R~$;V~6& zJkdSx(xG}akJ0>U34z*mFfAG^;o7L?R~OytU0>0+$E<8J8i<1^tmtxvI2j_01wl9! zT@q>EsuU71Od{ll3Ey3I+u;PDT|`u|43vaj`~Odog~fsCe(Q%TYidI=gT1^KCCL~B zmn4ekNjK-z4Z*0ZM;qE`*FxBTOZathv!Pq>=`J5mO<##j6t6UU>eJHyQa?$el@QYG zL*%4Kk|cN!7DSS8skQKyj)ID6Qe*IyZ7TcBS#yU)) zV48KeW_;RcPt%G`+~!$j0F0@k0|%m19(qKAs$ZTn5>zw#aEBzfJ5FzEjVruEk879H zE&XeN8XmlC)d))H8|s*4HfEW5XgFq>!8L*sek4%Bx$d);G-1|;ukGWo!Hcw23|VpG zb+@#Np*G&pBD>K`#IxmAT(xOPc5UhqE}v5=A4D+(70$lmM}_OLvbof;4Ymof5%|OR z;3&oRqu(gSI=}TuQmmA$7#2G9&c1Rmeq$RP`Hi_oelzl$wh?{y{HC|*5oY=0!Yq$4 zEh9|J2-7mcw2Uwt}1b8x25^T!MC#+&q;YBtAGRJnj4}yR5{+F;GmNu}YHPHZj5fQ z4i5T8a3~*}IRqf~LqQdmEAZ>c3HaWc>BRz>&9s}_kvyl-yG&^YXvA|`f1}YZXu8~I zq9P|29Ty_EzT*YMH%yvm64jU2IHvQ za~)CLm1<{myich+Fq_K@Qh)#HJ&?)Y+3+QW?F2qE%f|P7$^R1 z={5Zl&7B9YnZjY$+*wEN=*Zdm?|LetQBzUTXFzQ^YtauJtg06-QG#-;2lpmUl%h{1 zM+Upnl3$;A6Fpdc)q;|MPDuFuuU?Gf&cdp8HI@*S9y+yAHc<)FtTa`J zKqE^$B}KVm5Vg~o-86Ja5m66Cn>tb1OtHigW9@;1nE!H>X?6}LtHI9+;ayEMlE*x% zM@dZn*2+maMN93pq~bFpSExubC6jxjK|^y#M>oU0E5S%-#vHA7qP9pf2CZ-)A$Nw?%fv6XTv>j7xtCUgLFFkBg>X zq5P;$?U#-a#X(VlOg2bPP#4c9#82ggwrRiYj1#(bm|toUU553=GkSbTjLG+~y{OBM z?b51eI)sv6w1>xz7sXE$6@b3JdY!DV$vI%u={=PTfhaq%Wb&pVf#Xch{h~PT@rOC) znZH0o@<^k&?v$${pc=_0x~R_{b{bBj?m})KqdiPezC05M5Ep$G9e5=@{A^@{e%2&| z?xq9%X}wkw%EObjO!7VfND@M9Fw}_f5<=i>8E3n0v0vuL95-v@4%~e&PLJEjfD?wc zwl#*JWqyh)M1#=oJWM~D{P-BlpdKSAW#q68C;O(JO0wS*V(dSQ&5;7%x4=*NEDS(Z zmq%Zte8@|8qoMj;$2sx;%bp^(u>d(OpQS=!Nh!<2zlH)&g@|eXHm(@bCEx7Capx;H83HSX2-fim>7)UzVM$=#xv6EePg|#cMIaKt)6h6P`0mFj^vE2%JKu? z;lA1?iF!}fkUmq`>f|j0+9ZlKvr-DxBpM7LMoz!t-}9cH;%$RGSX#L0I1 zFm|-HZaXy0US0VIy#LkPO`{kf;&$ldD(IO_1x?&KINbqv!O3FTF)E2s8^tZBeWTq*rpAAIa zPBKNf`%td720gtv8hwYGa&Jh!Z z#aZd9osGNP8P5}cL8zUJ;fO__8jCvdj4hb>wt|zCKB4w@#94H7D;(o;t1_)$Q7qO za9*9GQYcTV4isfq8u}X4ysq6owXXA!qKLS2KyZ9Y;*RVtHTmL|0X1&SCF4IQSIWwE)09AY}tu4Hm z_)_k9ysdLka)n*U4ZrPw*~*>0{e874_;tTZ`m0s@C^KVRyFr{dx?LT;(oZe*ZHZ(H zFC*w!%O7O_R6B=aHuE290SA7T5uLxSxs?#cF}}7@DlL_cr%*_nz%dsxblULgt>47m zl>d-MaWW++bMTLuH@jHd9Qd5Umq#Fr%!iNgr2n1tM>Iv2D~rc@YG%jxae*IezW*V` zIraT3k8o!L&PV*-=XVCo?$6j+uK<06ZY{6pz+Js=@8cIf!1d3lsKC9l!2RUYFU5O= zUy_~@zgfGNAX&RzGCF{p3M6l$2eX2Ar~+XB=7$KqU3}UGVee4Axq+;cBquBIe3wU2 zWn${X2h@oAnc+ks@O!Bo%suauHWbGhkM%w~f{M6`eB*j_#g-tgza8HN!;FU(%SGGV zpLPL<=|JOFe$<`8Ro49^r+TwSN@iLluXI?DcObI9;bd{8Sq|}CO21UwsCG64-yk|* z6$Iv}23{gc;=@3VVO^i~Gaf&=X=>g9K1u#C0Zqw6XfF$&MX>2mgD?gqr%L!4R|O*o zv!fyX{MuR2aN#$K_N10RAvbq@&sNY;BI7UQs-SSU9g8 z%!~p`##NzF_NUE|3Yqy}5K^F;nBH$N**{m|d#|l;{m(49zt88?zYk#s2rRG?vP+1?tni zyR!0%atb%#OL2vfb!<~@AWrO36=ioVuj901*gCMCU{#=v*X7b(xx=@RA=d_$)> z&s@e%+dTI=>&U%0RsT!87@eDVX#_1-+Zz8sC$HmN&gm`TIPA9BJ;r>kIjrV6G&cp3 zl=T{M?K1xwe*bZrZVoybhQg!a<#6{nM2|!NoDx%m|Cyj_#LjIx1NXybWWd=iZ@Tic z?j47OE|pQH*}8Gh+v((3pjuZqZ3ih#%QR}gZ_W0Tey~&03Db@0Iut?`YXc|u>Vd*aeAWe@eu7Sf}aA@1tc5*d!h(-r(MK%xjed6U{=-1A#UZ{b_*DyOQoy@Xn+X{en>_89%AUUBY1(c(VaUQ}tRd zpHRoo+?h7U>o>a3kN3cH&#Fm7@i$};&pPEg=3LR~dx!6v(Vu9H^)r~7cJ*ZJdaSLL zW_PAvw9!U-W6#@CDZOoc0Xi*{5fha7yuOg?R1VM_epn-fquJ6s0$q&_(-LHZ%wqv z85gvgij8;UmDwCl&-Vk%I;DVL?I%OoIkrXP_{SN@XwN|?rCO~>{WjGAONxoPEV~(- zndc$~BLGcyR*3TQePsKuQv+$~0X>>HKu;ajk(CK&9rZ1T$@)(rrs zH*F*CSLYmH9eS*gpslR)rfmR(hDgplx~Zr#qm@RIVNg;s?Mct9Kbp;=Aa)Cqd3~O} z-bKQ4jrI5=rzhnmQQ$}6Au((dZxtQ$GzI%JvQ;f1;EKZ0d4#d}SlPG-z(x4SbY#6T zC;YR3B3)zM#5HGG7}`hms(^+}GYO^r`03X+mNf3%NSSQ`ZY0$ecLR!7?jm%IlCtI| z5O2*Dcz|CtUpsT6n-DqQR8}^tHPpYqO2Z-}Fu$--Cm|t_EB6hU`WMeJW{@thx8^5r zBfwxIP-iVL^>KIdur}6Z@jx8C&V1_~t&z4uLV!IU0YPSLj>{1lHdYZ%5_ zFHaaox*Z%H0Kv^v;f6g7V{J=fG<$TKsmNR7?my_0_z(In?NskR{tx;r|6n`du#ClI zn#kg^jJ<7TaG6izHSO?EPcuO|TCgIlJqME-d;IGJZ5Q*8+IZ`=!}HnV+T{_$Fld2| zt4ueKtfQ7-6~nBKTX$jP<7tCxOd$QsRy2(mdG-VYq4^tzAwU>jHv(KJCZYK?0(_M? zfgVb?YsY`nhPli$DH`=Sc`DmRR%nyfU&{F?Np z$<w%K2i<*QV&aN8; zQ%07*x}Ef+gT>T4d{+W9<(Ux^IrsCrJ{2$x?>=z{PdQf%TlIW8B>&YVS8Rk@{97@t zaip35oP&!gQpmC34TdJGe`Cao`U-eMXx}(9Gr^Yx(}Wh?tj{vnI|#thP2iY4y#38>LNai~f-Y5O^DYWsNCVUZ?D>0B&FF>~_w&PUWpTIVdgCj5F_l@JMT8lW{RDxhDQws+gi(aW%&`HeJv znr6d$ZEArXrInnQL>gLXvj&V4dCML)grGQFylt7E#M@qGC$EMjfvu%fAI1)EAcMZx zLYru;jjkQ2W_|4tZ5(%@PhqNA1u&RJL@(o=Zg+BX*HuX5;|r>zjVOuNtExpsnhd4# zz*U!UwJfo+PWOMb$)lR(oLt;@HehKNVuf_10myiM@y}=E%gfhrg|g77PjJ*tQMG=c zVH{I^pmzEuqna;y?k0F#+IqA5I&VpNejZRPHC=4fe2c75V2NIN#NA`I>D~V0-x>jH zDR<{zTKUO(XCq`i6J0ArfS;Ej{4U-Y|)Cz1R`Y+E@qxyn)f_gl# z^lCeqqdRV#G{qhLdih@NR54*`m4M%}YpZPALn|^DxX#mMebWDJ!UR?-wvE<$Cx>E0 zbrFQ!jyl`OOc%*A7JPqCL;=cKE-}$%+=uO!>t*F(mzj=nuj4viitSF+!vwVvmeW|S z1-#j8eKCD;g}Z&A598UDy0&9Jly_F)+b|>sP%j=1^sFJb*~k0gKOCDvnVQtsI?W<6 zS^41_8|$!pD!3S2hK@;gBFVQQLGVfp@86*LVnS$+1~vpMUWX4aiVgY=Vqy$c1>IEzjX~F5(HjY}rUWxS zZhv3;&7z-b>&LxdDd~$Gb{EQL{}b^o-?o~|p)$@J#It%WXP6Td$)TJM{lY8%_X)-| zr&x;$1GncO=^3KX>|!gue|#HO8=>l%I%N~eGAM~fzxbrx@%z#b3P{T-I?aL)Yzv?E z3h&428@4GY7vDX>mMeb=znn|;;Gg78^Q~!@Tky~%9!1WaV_Pe5aqT*eiLSg-baIYk zv6@E>re^Yhyc#~akrm9fhAKw zO&t7@vke_Uw@`H=D#PubbT6~t+}POI&@k|``u@ax+H0EYW&H7d+}KFBF!(;0!1RB= zoN8j|{dlZu63Fp&2aISKeErVldmdt{Nq~tK$jL<(O5T3TQT611{Q^!mKlc5R`fNtC z1ld!-RHcj7B8sfn+_f6iV8aB2Y-x7P?$ykd8u+sF@sQp9x?6jhCPlue*wIa)D}}}k zkpZT*>G9kpPnbSRkzM3jS4iwq4a%}jboB+@>j}Jz%Ji3=YZ0Lv=plpJ`qiGmw08+; zA&0U#%tIouXppTk^Jb$HlNdJ=+E#LXSLbUwRF{6^39fu&QMB!pscq1g8P0Ffng48-;x)pUzle&IefJ*5 z(wFs#AFGi#w{U`ugk|XS2Swj~I~q0prLYB?wix_^5@g#??eF#DPz{gWPXlI$XE!&0 zMgL0E@@DjsUw=OtiT`K7;n!0_xHhKz5miOiQjnd5^%m*XO>W+`a=0n~RjeUPaCw_q zhJ)Pc9zohd!VH4cn5((5no1oq*TT!1WC(LaEZlhs0c~MJ-26-CEoi|%wYg;K(1DPG zE~{2Md!RH|Gam>6UDy`{qh54IY{$*I53Y6#M*g>j8cQu(g;Um5{hZG^D~WAFo=i`+ zUNTKJQ4~EW=P*Y;|7x1BT;rl_Fi*>CortfE6>ohe4}BhY)9{t9bvvXy1@*>WR)e9| zA>osiR@?_*v}@BJZ4Lb#qQp*t@o^wX_mX6BE4@lg9larKGlpOggZ8h`qCf3O@L5im679krr;lHet0f5~16A zAydsM^D-X^9*mO~Qm#e%mKP#;B5urFU%fzDcgM}u-L`t5rg(qu+K)AA-n0H?)yLSP z*5*LAhHypI76V>zk^n$?SqB}rmF5Lq+W;Rq?c6<#OwRSpUI7X7JnhQp@9G~o9JQI! z%kO~ZB)-gaT{*T>R^mP-b%T_ei#gf9zVv70@gsRnh4B*<0-~@5Rwj=Vdm4{Yo|I{4 zc)WExg4*bxtVGGV5X&ZaUrLcEv=brB6%2b(wQ>owbC5qJ9anRZAJ8-TPIR5)!;hX9 z>`(Szm;*r(-_5v|3fPP?kw{dp)X#^mLcS1#Uq|KNGX&zsg-JRPRjeQs zY>R7VTXVOgg1<6)UYEU~p0su6h3(jvv<=4QbvlJD;%>ZPHP-pEh}W6Nq)bj|L?G4)mqx3k>y$x%Pb8Jx zBa>AA9+gNKPsCS9DiN~`B?b4d*)v7ZY4F+VN50~#sr~Drt%bv0DF~bMofbgT60mq@ zZ%;1(Y9sig*C(R*w$$}$?k>+FHmo+lS{8eBzmLS6N7x?rq}plg9~AH)f#rN2AuhX*>Lde2KyG3ALWkFEF)ro(0F>OT` z`2p<^mj24Z9L>YXDpy%Gx7Q;wsN~-t1{Sd!02bD=AoRy<)tYbRL8B_?2$kQQDiRClkrMsy@W{5%ukct_`2vebg?$z z7ro5;GID=%7An>Cc(QY+nKHCh^V({=r)K}Pn~4fu3~aJG!CM?C0d1Cgr_w>XXeou8 zNM)w2>6YN3N``}%qrWTIkmdQoEzRJdn-xA$;J8a8q`v6(c|%s7%WnMvdoI1n2;&!Y z7{@`(3btnQNFec#<7$qTu=MfQL6eoYYw?eU>b z7~kc?khIv#$6y*)wEl$XiKsv<>0r13SKdf}CjJLY*6T4By13VMCMG5f>hF@fY|5W~M)3%E>GVJOSn{O~+OpEVOM{Uwa!;je95+l!*V!1^ z)_wfN`=b$(U3*NmZ^xg3Ols&TVa5B1GW(FGJp`=21I1tX$sD9?{)WX&k)PqQ6vUDK zV+jx?kMReXqow?e1^$+elV0%(SAPhx_W_2Rap1p#=viHeAya-sso*J-_c!lE zgJ)pe)We}8)GLHeL1q9b#lU2@3Q`2C4S6FOD3ORal8F55km zch)GdGh4T-VCRZX_T?5FF;Bb*;-t__02bAdtdBvp=^sy_W*T#(*y{_A(Y#hg5~U*?ucdhfEbwW>6%pUb1Ca!8 zOWc~a5>y-rrorN#_&COk@R>cRNvK&n7C?r!ey^3i`VcP406>?Z*CU}L4i-Rt z^5Gzge6TkUQ7fpKnK=*;{~s%!m?)tlbT}M}j3PSBz_!K8GS3o&_)}k@e~hUj3>kNK zi$vCEh}SB|2qAsE1V@aO6YOSb)au*UJRxc@yj0FaGGh$(3xk&HPW|4q^|blZ8h)9Z z$e6K`X$$hd2Z|h}x|P``6_>jP0{7xzAFBYm#kP1Ho?*Ctf-GfzEc*O5$q{IGdgj!l z^?Rc!pXEhQUXK1Wr7q#7lGg_xBIX@6>8G%3Q+Z#6My(B0nL8PQ)v_z6sbmyzvH4Gh!K=YORbuj<72gf{j zQWoCb0XM@?h8F`PkN1N)NNo-l#ySt&WX9(U3x#jM$w0#{8Z4bN7z4(Xos9dB?Z_|o zo|C=s0#HPIF*N%5{U6@wYsWyh6uV5$iFaau^ZcnUOPmnc5e>}+Fbl0d)d4H&E6^tF zz0+d?sZgybv8Y}KwSUTLFL;7G(Ddg)jVR1f-xC($izy&wjm~}fY$N1r3tMg0seLu~ zJ`O(<77j$bEw4QDCwn+}oWk@H<0nuZctq*G@i7Tg^4=uJ&Q~4C&RqXFO6*9EokMh6 zH~$x~Wf!6wmmo7o0=$BU-X~#M3fP(KRvfG4%tnR#Qr}AivOva<>trC1?7JkTKoh<+ zOhlwu4#C>M5zew3_W}WS{a7HUDU)fa$m=!?V?Cy^a2zIAdYpSedyrPpyFZ_A(vB2Z zJY(>B$oit2`Zq&RbLhxhO3Tm|N!Gh+(SX;?6LYW`B95V5&f#S8l`g)0u96lTpgoWGuhsb>q=ahU#rs z4&rPN(CIILMq4`Xjlf)M;b06cmCRKwNb(po6c=0qXi0HGbb-GOwwySu7coLffzgd^ z4`FQBUN$q^fw3`pk-DkXl}sip`)s+0*V7{oTXo1lah-1$8wA0wQgeQp!JM@kES;gA zT?{U^xA|paUB|{$@+l;ES*2_wNdrgQ1#SKf~>(nH_u!A1yw~-mGy&0$6MKHxA)|i-H4V^l~*xmqFNyJ)l($}tt%*6fbTkvA` zrUS$~RLv&pfll4~{F7d{uA3ikV`y+iBlMr;g$&X|$$7Yq>}~JkQBBsg}7Mu#rjf6(-_~fS7KVbk(PACotZx ztGI-(4ysdU6D%W38Uqxj2FCoD+H&UtDYoZGB2|pYOt;7wZA#*L+%~T}Nd!1kF?C(r z(sQ^PDD!^)eU~&uSLN#yI0H9l2m#8y@Rh*YzU7?_v)0PYMyQifuzgV*>0INTVX3Tu z4n~2mYbEiTle)-`J+Ck9Zhc&V>sMzKZV3JCG(=+*zhzz1y(U9Omp>D%DJSo>LOE_? z_|5hnrwyBW<4`q83APg`uAhVA#U9p>fJwtjrHxHxZ|!Grn=7XuH^inq8GRG5A2V0T zpvUHOwm;WBoF9Ws_t{gWJK6&fg+gfc8M?L4SLFW18BlQt7VO5xN%M-X3GAJ-}J zb_4(PIhzxVgEByBOL7873L#wSbAAcmrsix-{$(_rl?8+)|DNocvU-Md$zn>s>}=#Y z!ofX_HH7}fGTv-8P7DGH#Dm*Q5XyU?rL7mDMZmW%}Ez}xjT|)ndTMN z_pR>u-v_O*u3M!oBO!C%NWboB$NHX@9w{?k8sl*Tm|D-lV4kP_)|bC77N@ssokcOU z?TcGACcAdv{I&uDp~HBjQ^jn#EfW*Oa~+DM#Tl=)lGumDY*Eto@cht15{@wH+m5v{ zmKf&Ww36Q2)wgRxEwL(P((Ek7Z=qU{Vm14~8kO#sysA^|c6(Ydl;K^I1HAS<3})j= z#U=!Mub*~$ik(qCL!rDvfr2d#E~C0>6^`Rv?(96df3AC>ZGZfj`LgqNE(KJ5gM^y> z&Y=9Aw#>dRql+{0F;-?RIzRqA5kB+XuQcv`n#9m8C1Z510!Z7O*(W$hPD|us8cKGb zZCK~4E>cx-$x=|0W{y14oC z%Z@Uh(ybLz2B&)ImX)%TNU3ZukX4vzB##bjLGBBk@0#YIFLi>4H6FI+)l|>gOg#xwl zBFqa}dR(O@vedP}tR*{mah8nE>kP&sN3Os2 zCaDua`Jzx^DNDErOx+Z@f4e$D6Dh@p{|tBKb2|GiVQ@X!dDFw4)$lmg^s_u zaSmxhFyk5N|7v;*gHS8dmr;&@(Dkq^=rdJda^LoFSHH8DgjH-+f?fcia zJL3*YJzq1ZF~?x;!qXU0FBjV}aqaFJUH6USdS|2ue)Ow5N81vT8%G_VPG9zG#?H-4 zS5?(2;mqN8r0edL)aF*ZVCx&O}lD28Ns_s5`L*M(**j5r5hIAe| zap+HWIhna`6fudgJcnoM%>k~3g@uKI?|aGnox!yK9RDZC_vi6c-E8m2)S18MrosiFR<|NkH=%^@?HD=wLG4ymQfDucbManFo9NKGhye2_N2)@9G*(wJdxjsG8QKiT48I@Q+-BtfoSQsf>>b-&1uG#KZc0|Fv zTxJolIcT1~?ze}htI@i)9BeX^@`})zy;JNxLF%$3(YDaa(GlEBEfw??-$shA+K#O! z8D*Xr6mb=8Uqf#lYSMqfL*gN34PGnHks&;S1J=(uu_d6Lo%xeebs=@?;6>IhFhLQsFe9P7auZpmWl8kpnxrz7 zl7%or@=o#;^KRxSDm9A=JNn7t_}+J`J-&@r0O3Je=Av1ctoqSd7Tzar> zHaOY}C+Ril0`nLrPaW0EtLe4PtlwY~ci7V~o6yx3Q|l8F$a*U$wT?d=OI2J4=2^Ol zH&|#ViWma&thg3XaofP$?23W+t9Bby5WZJt`MvB}E5Cbl{$?eT>8t3UURquKQj=*E zrY8P4wbq&zM6ABOZn4TX`|I90_yN=_Q;c^G(&A>iuH%XYlHW<&aboI4US)^#3BmC9Am(EuS_n$p)2R@PfKwIToJd427q$%JC@ueS%}ST!ro_ z!_`-9YNP@SE3WAlh9KWH8`(fItL9s&XMMqw0o}`)Pdai#s0=nu)n{5@hE}2)ScJ>A z1wE!>K?WC}wu>zarVPNv)vMS2BkvMK)&RE?3u;bS6nR%9$#LSNL!JHGLOD5P0n;Q)zUs%S#+)mM{m+!Q zBYPBCy)jpaw?)3H6yYPy8DH9cy+JEj9WUhElH2=1H2Bd;ok2H= z)Byu<{x>Ix@-Q><6OoF+_?M7}&WB#Uw|e!7y0KUZyb0A=RK zCbXQH*<_-F6e>G3fT5O(3t867oM#K0ZC{&i@OOwl49HWz z>|x*%`*XPMJm_?qoZD@QjN)J@A@doWWhCjQ-X(7jSns zzpjl+J_8+#?BqjUQM8)a{8whIDboZae`lkUin;d!Hg}Mm^h!a92>Zc%!ydAKoTkzv zyuujL(Mj6@iGF3ijTFI3_&VnX%pXolg2(OadYE;Q;mhLNwdi%KQnt2#nUB7uQxQY@ zUg4LD8Z$%^Z{FEHe zEfFu3Wwe%Hc_Qg8h$asCf#2)Ruv12dTQRNIncJWfJb?5=#9)Gu4La8UMr#W&p^!7h z`cTgmH7%`cOILbtxef?p5XZ7vwY5`Pi|4>?M51%mMEtNFSHB&oXlbwnl@u>n4O+q8 ztO+%`3Yhe*`pRbX8oc#nzf2Z$7OW~o6aXd4YU#?FB&k#loqtq|Mj8z3Qr-M|xS z4SpIE{jDJwiDq4L=d^7gzbPKDVZ_bJA!(I7Q;x z_p@3Q(Se!62Ia}-<>ZFC+rZ0wzgsA=;HrGGlB;Q%B|r0`iS6Johk{jdT%p(BhoLJ$RfTnA)=FN`uwE}4`tY*mrqk32Gd{;rPI_{YMf ziL~bG;qec4-S%|X=WC93-AkWao@KAi5Ap)L7dk+KyXT`^zwRZEX8%_n;Q1PsTYh6K zcS2b#x8iYtIR1F+|FiRteD6l*f_Ecz$V<8Rq6)3Cxxc6W8fp$fq7i04xt0-T|Dx%j%1eHFmLzyJ92mpOy5%A; zu^h$9YRAeV16aw{2kYI}MxL)DR8%;j&D?)W^t;W+wRpuNvw3%I%7GUQtN<8`WNNCi ziqmJ9yvV{kkVV#lSK@g>OdUg(zz3!^2TAp`7rT96q z-`uw*zC+v_gbH?EUmSJNaBSm+gKd>9+x1}J+vE!H#R+5$dw0T$S?{j)^V#sT0KEF0 z97{!zC~x?pqVwxWKntca3FE?!Dw0M~2s` z(JT4ju$=;imIsdn9-wy@LViP_rbpPIM+ng*TDYtzD;cq-O`mluRF_uIU|PT|XCIAo zo6h8NSPI!+bf3m4Bk3Nmo7TJx(HQo~o^R6e98R-I)qY>CKHEKF+6B7vWjOrb4@X

|n**B`~R;{G#1Yc)`Sq8Dmj7aQ2a`c0*Wiq?Ix$)HSN&gmk`8Uy7|u2K<;wX1mO)_F`2J9m+| z#qxO!POegI>}>f0HXCh~oi(y=W`$w-l;S6r z>o#n=TMg2FT8Z?k?EnJM-p#OfyW`|>)S7GD89wFYHL_cH+sEP0dP-#G()IU`hk%t( zp`Bz{jWY*{8k$5@Tn^I;q-t$e&CW7mKU!{zDs_hA>{7ej>Nv{kub(s}QZqx*7M??_ zJ77}8sAom98W@qL99ql;gNykQIdP=5ynS^yKsWP~AO@e_cg1@|A`*&IOst{14r&RHaHf{iM{i0!V!?Zs~3^p2{d1L2}FIf+^ z=O>Y@F&d&Re&bpew`pTzb=q8mYaPVuh8L|W#G}FLW`3_r2GCI|$QwY>I*vBeM2$A( z%D{pV)De3W&G+6JzV0c1H8z$eq$(=8zt4UPZ;>gbcUEv714AB>aLZtP>7_N?_Bb!& zS`2R2+R$q=2JEi3(DBB7+R5USTTXGI6?;qM1<>HDlf|hRw5p*p+|ak2kSgolqtL5G zLa`kPSTo-V^r(`cfn^jciz$994$_VEKiO7O`6_O&nhe|W$#<{PH)!9n!5{YO8iQ^d zJ+ubwsDQn6wT8kNE$WLbJSgV+U%b)IW=S~R31%u# zZ|0fevMgoXi&-c(8S1+p*$p-N(&6NX5-x&CB7VUT7)AS|aefpDE_$={D&G#}_&gj7Rq}bdFr@MRCAuBV8k6`2y8s>kd#5BYlyF2w z%`53)Q2rn(W*NX4s{u(6fbs`#(nFfbKvWZ5U0xc4i zv~$L0sGC)?IeG2F(<>b5#bF6bCm@W#TfzZ&YlLmBgm7wH8yi zITitC@mYL_m$cB_6}z3i6Ar}A6+2|>oEb|GqG-y{72CCW!ED~TXexh2Jbj=qc8ZXH z)xv_ZuE5R(}f{ER{2%#BHL6}m%2)-qTlNq@IN5=_Es1Y%CDPnuT$>^6; z1Gdk=s{~q`QPi#eerfNKPv!RWGkzVHoRU7JPOphzNj5=L-_Zw0tl2qj&=JvQJtG2D z&{09f$*d+Ag(6s)!^)^MCy<6seGr@G;F?jDRekWGh3Q>W0yjB6c4BPw*u><~5JWW5c*Y%)}Uooh-lkR-eaXLj%g}n3}dxb;wg!MHSPdw1+j$Z{G`W;LNLT*YH&DVP2yGLP|&|9zD&V58gqPl8ssIseli zRoTykT&4dTQ60kaza_?`I`YnaA! zxc<7X;QcQFo&v7TVSWSo?6}z2{}MKk%dh|INy9-pKagd9H*0d*+WWYU_ujWhn2EpAthHbO1NNa~@um&RAfsKd+`{ zC}9u+pMHLReZVH@hgg$lD&yS_?HlTH5wdq+3euMsRO6}~agD_1x>|E#t2Wng|9oFO0_)*^?FRL*WD z_Xfe$x^yQPbS^=(gAiH@T86v&DVUtr6$+8ZTPIi4igLBA&}*=hbKeV?db+%U6-B0> zEy91M4Ou1e=4ze*YG=t`20en^iS}1)ISCD=gqNz>FJ?wFP5R{V;=)RwpQ+@F^(3+y zc!_rHciFs9x7x=as?^sa;%W1}UIL$M;kc|DAojY-;x0|c2zE$J267VQEVN?o-7dD| zpmE(oWY6@QPD$oH9jef}cDiG~ib}YydZciUybEF|!}l}pnG+3^S)S!Sg8onMXce@vS%Uak?+I*2DJ?HQsMNrL!Fag)etR20a(aq` zFeZLwJ5R+Ywq~x@oK;3j+u9_cMK`llhJA*8!I=G8MYIFJtV?4&ZiOnd3&qw)BAV%d zcBDgPTrr>0J>O$YqQw1Cqu>=gho9mo~4eea- zvUByZ{#Yy#a+U6R$caDu^6;G<>bx%CI}yG ze)hU(Zjm7i0Uxy9i#yO|L^lNJU~6MG*@AXXE727${(`cDTLkJMmHHD~-I!fzN-@%4 zLcjeTz3bwvt5}M_9t_n+DYY~f6{COaihY1st&2wLfku}|CNqz&wMo=Z!`3=vI~fEE z^yc9eYk8NlGh43}ffcGVM-gVVi=BW^lF?XHI-Jd_%K{(FgB*0fAI->d7N5^L)Q41w z&*i2HG2dv}&7k4NIIieYRoGYdg*E79b9N$@9CeZ&wp+{dZz30-U*2urc2$aePJ&Ld zLt;SFt(Toq$_s|;xbq0CW2C#t=(0~d)5S_Yvkg|R6(E0y-n<7n5S>cZu5QvhD0C@1 zgwKKwx_;Rw9f!6%ps0ofhnshAQ+I*qTjuK>8zmx+vVRM{MKB{L z?64UXxd*51w2#Bb=jX+Y9(<#uT_To^gg2#ObWh*x)jI&^dj9n{ugAMqW9NRiLkq^n zX$st6DWzEDnFIr^nXQVSRJ2fl0O;xW0(g0^!UU3Q_FPSL;WeccmnW!6BX9K#iS+EP zsyeoUlpGt!F=}tsy(HD_2L?&z)kt|OYuKYG|E7Kt){?)LP_~{Og@cqXJLh6}w)}aM zzzQU^O4O`915Ex=$XP(ea%9k>e4#1tRjg`a>|dD~G@L3`VeZxO(M^T~#-?J|uk z#nyvpac6N=EMpd9cy5!@)@j!-bTr4-LjBF})@4f}nE;MqC0@)5m%o@21S=-}PvFQp zDlhfQ+}WtOLA!kLQ9cich(oGjFnLppkIj06^q>me`~qxiiHWLhVUoq3Qobd~^eIjgjNS1MmP%i0cQ#)>W}QRrx6}LS z;fq`X?sF1?VT#^;DM?xAEv0y6Z2Mz{Pk@tcD2FMdL4#;w@^|9n_MS_Km-K=rQipj`%9|< zlkr$@5hHX0JXIb(F_|EfHn18x?bk%0bU8Z1??U-EUJhjCeoRCJ?%TK*_TznDk~i!A z0~Ufk=HY4^jz8LPewHe0SgmS%llKh3&j{YfhhIJ>MxLQhcN3>k(?k7*r}+cw_K7Ek z3qo^v1B}9^nplFaP=aNdeXQ4Gf&P$(!b8kuO)f&gerQs4)GY!{{$22HprT1P9vIDA z{cMjssGD@>x|FY8a6$9UMEVi#Pd;RubOb~M;Rt`gUi>7}T-N!9-sg=*BBY07Lx1zT ztxje5_7KBdeU7UB2}wQ`_zm<3vY#@@YTx~bfcSItiTU$3H#B5mZvycA{}J+!O`7#h zvE&ePWwU#Q_@04YL)Ge1KMU)z`5 z;E;K>Z6u|E74R$O=qHPVG~tMqc44ScMpU*-PJMWfgy~_A8u^ei#R>|PwY=f(8bZ}w zG1N3))!4bDJ4N#Lc{0X3rEFcKrWa-C{Q*KU zQ&vh@Kx+wf_VSBVMaIiCkWcQ8;5vCZN7Hg_;Bx#WeNU=>QZpn*!|)K2pF|i&rqAan z!O7h!BS@Y!FlevFpxfk}l?U`ZmN|*(-&U;iS+~FCrFj$3Lt5vdRjq0NNj)*9kFu#@ zKz&ehDM_jQ7Zb}hV)ICTfJg|^A2)s5`#l?A*a=@cW2Ej`_*A-6Qqp!x@ANUzl_S&s z(J9(1w|;j-7^N7ON*u%|_WOBuD)7tA_49FdR^BwR>-YY+FsJgiuP|mn@O$xde|mUW zT%6zQ|9m}j9BpuRUnkID>}v7ze7T@-G{TT58W-R9W1{+sos;+#9v>cWh)oL)_ao$@3v~1pW{SxmevKiP++F5wqm}gV^)^FO|3CGt%*K(U+$$G*T%c^n(vQEL!3Ul(XjoM)}1$af+KI?@+Y`iW3eA)yowkfl=WTMM=+wVVC|-1| zTWRgP5LMgl;c>1ngs7foMk?mgLj12+(+nr%6q;3CxCjn9ST#95l-$ zYyDE^(6}N(QwhB~_3xLwL-f+?Jn8_1CI1ng6gXL+*L@sm4J$F` zEQ~1seitF7T?85zV%);!gvAM8I>Snm5SH2Skz6=pEM*mrnc27=*S9%{PApuhs$lIu zxU4a9-Rfs3WhTek`Gx)S{rj{)iph|8Lab;bIr^>YoIg?;o~?mQ1Z5Nvm20XwStOA` z9a!~_v=ov~C7|>s+WT#Yq_ogOoW&Icct2hqK@rk%3w%f0tBYJ>9JEV+tF_Wm(Mh7_oYp~=zx0*u>=#)u5cN}sq{Lkg_CuzFD}z2rr*flO^|3(iBE z2L@t~t1a~D-kwv|C6Jo^`+vN}YDP2Hy<|gCdH(Ls&hKQboO^Z7iGCJ)yE zDU*F9oT1}JIBf(y2*ho2*EmC1&O~Sig{9ZVETE&ct~vJdw^-p_7^EB$ivZv0t(%W7 z^CvUXuB9v`U&F*WLY1AZ6HP`CfDR$xm7Vw1tJ@#hb=_ck?J9S63x7WeLk*$SiUsm^ zFNU16_?MW9+G&=y8YDy}h%7UJhP0r;HeP?bZ1y{yZM7&+d&o#T zSV$YM!5z54n2n?e*cL`j&NQdC<#^Y}R+5_}9|essQmS9jc&{Qu&6lwidT&N)tI+a0Wc5k1*{M7CfCe3;{5jXWBz0%SduOge)(jZO z)WEBJ8gc@{GI#YOwsnft)pMzcCR!C7#2PgyuK+94S!h0{A360T*4L^oi8QBlD7D`D zF|P|xBYXupX4$}~fGK+{fGrU3`#Ab+27Rpiz&#spyO%vj`oTJH3$FWucXgAFskI^7 zB0VffKXTRHjvCX&A+COf%YgfH-hHLB%)ZE-RKx7<0G7i2RkOsfWiV>?FS;7aEjTih zkr3h`IFAK9SbiI{m0Ig`L>GEr)5kjQBJ6ZA7>i}_AKIj&(oDt_{OTS2?_T$}(T-}b zI={r#h^Xt8T0bVnw%ts2Bdnyrt@sU5g-^O8n|?i(~n!gZHqu|^j+0r3)EFaeRo>`(5yCgB7WKBvYYrQG4dndbyZ z76!yr%tPB)7~xD%9x}S$ko?+yCn>@v7*7ekhgX?Ae0=rAVy+M2y^w+&$jNVIh#({t zzq$DsTFPWQEtmYQkHFxNNoKI5UqoFtWXQF(xDXABo!Oy@0LNgDz-L;AG~3GDJ;TIG zXY6j)OO=o~XmCDxsW1+3BF=JHS}b9FfyOB*28yqlx{NhwdOaD~9q^#!E|F3cRtk|k`H&w9Gs7~uO7OfzjX_&D)LCfEt{9E${$8ypimgPqf#jj)q!?L$7`^4nPZro*; zc`BSO^ly{z3@Jegwn?LX*5MEfHv4Lu+&z<3(wr)ReOP=mI(&kcqj?NvtBu|fhm6os zlEn_o3a3FGWpR~pXe&$Bj-^f@bDRF*MeDxF7I^FP_) z&5Sz`BC$%3S#ESSf^2lGFlQ|jkyk0jd*rvuc^6Si0fJY=g+drYJP|@DP7bzJR_)Oa za+@w-9K^MQo?%tkHjKr)Kq{WCq!Zhz&jxr>!qR9)Ld%UAawxsr%h}UkyR?T*i)d+4 zkTqPm0)?nzC++|UC_x!`|IOtCE*-!r4KPjK};Xk zite$-2}t}CWA_rHdB+*v=Y|BPDflS2{?(t~#;UcLyC%)uxO>jHH})aV@R!J4@7N@4 z;bV79H#Y{jCpue99H)6cQSti&{90Gp=jr%hF_R$2-ig0CH}R$ZzF^b(k8RIV`*yK`@QTb2A$=1=LJ6 zF9vdv=GUo)KwbpBnAMz=>imH}DP06jsOt`~D@Wdzy}O~2PD|{|6#yoj#`_dD&Us+YI@nd=AalBaQD;20> zr(zaiU`-@n>PbXONYE?{tyZSjIq|!!$cX@nFNmTT5-Y((tDY*=bX^A$qHa{%*jw#w zWJNVnBj|l`(&Q2tKC-Oh@GEh)Y4IKsLC;w+VT*NeI!E=B*s(Kumwb-={37e3n0G+L zT}%@gEM|6mqC1xq>ozG#2AXGkzP7%t2a&Eod1+vCC5B8>@^~Zt4xAP)|igNwKbeQ&b{`1YWT$NT~%1YvH(2k@EQUYE9d}LKJf*VvwyM(To`xDa=AE z!V#o(ugtk&KF3fEG6-EWaX>pd)BBgbps`v?mr<%4T9B5NUU|8YW)P;OjVI|}za9`N ztMBj6o*0bb$8ge>>g?||gS61UlcwC4w7ohZ`Kij307##okB{rA$6v4eM*pdv{)xss zf;1vnUNDal_C&0#eaO6Ng@CJVj?HQDG;O(Pk*GIceV zKQ}S(dEFJG22u%`?eW(Yjx=OzH1&QEdS--{*d#8$n)aEIwIa$c*8q*+Q_J7h(!H$R zX`Au5AUTDv=l(r*^*pU_*?WMtbO@TA_b+fi6WBw3#OPZ=Mu3axxy>w^1vxpr{YbfN zn;#+5RIN~oK=Le)n4gTzZR<(7oF~|L1kOT?kM=YK!l@4!CxcC3g{fY?ylq#lyWanM z(p%Tt0>~j1Y;b&OZX{G|xhIq6BE*GVv5^9)UHi7Lt>0~GaKd=}(X!WD2^LRWnmE(G zh|MWiA=z;q14PlC_kmKrNczJ#(c31}A3i}z1*tulw>m^K3ihv$3RwY9p?W$w4WKD_ z6f8xe)hGnJb(RIs0m2+l%`ewOKla~`Bgw(laBK5JeW_g4-u6z`%?gCQmd;G1pV*E> z`V#y+Qd827C9OFWOONAhyudVdU&)V&CH&W1+(NsqfrIu^Zjd~07>&vF;;kA>E1((= ztdp?mW|W9&XSkL(#PHI0WR~MK2prEz=zWJv?Jfih@#uC;2Y|N!&2(&sOT9P7%6DSgpK96vUupl( z6LA*2%5sQOzhd;B;>mKT$|sX)M@cVgDMMYyd_)mG+fXUqYM17CIhW8O>$7jH+kx9L z%dvV0`UoIU_$!V-$CRiHZM>cn*0MDdSX~Zc<@0Vd)A=Nf;D=#ng3efTdlU z!NBPaezq5x-e?~4E*n?$9(gx0GxbBcMdR{2YchgtY`O6MS-q9$JG+`KkGzJ=SJ@-} znk?(Su&j*%J*>N<8Z4B=%Vnq#gqL}bYIqh(25V(xGFySBGC|z!d$ff#ap`IDy)3Dw zWT%*9E>$-0fnKGVW_|>b{jC4X>Gp z%p)#Ld2Fa{x+2&ohCA>{w&H@B9mBUGh8otDdT^t<#S>n<#Xdzo6QHOYDd(rg`3Q_c zag2j;3w0@SeC}cRex51Rt8b?J{P;d1yWhx?#_#V}{h!<}fmRyJO8lUz)Pgu9G~))Q zn;I`!J}#9PIHk@!H_+Lvw6s4u47ki&DRKq!cH3-l*7rqo($?^WYhp8?;O zx!SsgaujrHgSDiW?AWF(cs(6PvP9wlr?seyR|h{{;oFx`>|Ei{_XBSd$Fw_$KXtBb zck{m*h<1@zMK0H*LSe+h_%kSR0HDzeH3(uJCiU2|ti!9I?&K@|nz;N-Xu}mHzsI;% zQK|%lPn?q~%Ke^OqSGAxXeMRXpDJGa3e7YP`v-H6Net_ zku6M+o?4RiD{L?I*NyW$6RKcE)X4~~vXa0&2$OSyz!C4Lx$KAlsf*poS{0V1lJ^ze zY&95aXd6z-ULHrOBH#B zVHnkSuvxcE<*3mg_ew#%qqIQBF*3eS7U~PUDJ&$LtZix*CH3x7zOOUkzFVB8+t>X% ze!qOZS4FR$v1tsaZqT;GmNegfxDankkM*ZaQLWYdDV|gIW9X@c9g=zqLR26R20g`; zCI_eY(9wMTh$hScJh7tja^Sh`g=bL#cqUJ41_SBUp;!ENzSe{d!95iFmTNomTHy~_ z>v8=oxRid5@80)}>lV)IIjrnnV``{7FBt^W$^}u3 zYZd}KpAJ%CcWYturkK2|c0%>I7{EbFX3?^lt5^}Uw*6g4@-W@-}%(7`_mpRP%W6yFQ z-HQ`jx9SVXjSBdHHsknJf6l9vJyVb%y(&NAH@&m_=N(bNc+z*F^*zYu(}JF_N1S^U zn!%wlh|7k-nNMv>$6?>qV~f6m`H5;+l=hG2X}X0tlX=A{TI_aS)TVcSpYknq>53Wr z#y9#}E%TP$nynSiDU1I7=+G zzWUrK2c+!md3#0UA#ExNAz+|dGq`lir>%SqrX|ZP63jDcs^x?F%{vESg6gT=D-I$U zwII6`U2&4H4-hcsdPwKy6K@WguIMVpYTFy;ns2?FCHOPc5aD4C1{w2EhB=eO8mlo9 z+{1)$)X{)-T4wc*)*q5=biBcHYUUg4lXtjvKKIiEB#FV9rU+i9X)mdBcMj!lOxQ1@ zD07UD1*pOX(n5bAXx;CqH-^l8r>;w}%$xYCSsmaI8rFC7Ql%e~8V_=zwG8Sdja8Un= zw*Z|S&jJPbwX%d>a=0*TDT6?z>|;&Pk^=&YhscA3blf1lVl0M=?&>_mWqG4;QpoRx zVxjPHnaO?SY~rBk7YG0uQ2|%m*vaTCGfaWQUk;>)i5GoexsKx15lbnBR_?QPRssn2)uP+Pw;Qj0_;cENj{ID zW(jXSJeZWcg*F*920TZd!TX|xDoBc%Oc?bzgNY8Q{&mR1-Mo&1dY+#Kq3Y=trkT%L zXR=q&*d+&V3ZkY{rA1cWz&i&j1}(rGOXE zOvAyHAmeqrI1@+`)!@u~e4muT`;6I`D4KxtK4v?d?ds+l?Vo+-z4SP+e*nj&}asQlXQzIZwlCBP9|7hb8O;oOU_)4A@tp|R6Q zOVw(1{RqIU7<2r*Me%hHE70*s&%W`-h{#kgeIbI!poz?f;)4})bj$r*OfIf@>a44s z%dQ);9UJXHeB&8BkCMg;(mQigFqfIVolk6IofQRy^0s9A7D*t=$H-}^2W9P*O{rpK zf#k|gD)kFzU?UsZHBeEpmJck&&dc<0NHz7S?yqg^IrX$+EUD*h@AdPXPM@`Y%FtV( z)`W~3O^EeX?TAn@VJZQ0l5~eEMJGbcUc|x^nZ=X&YH}pxSY+mL!S?Um>6M*W*HXu) zK+(bh)U7pJ5sqxqJ5bA?#JxJ`0qFO|1_Jl><~)ibjW&zUDS z_^v>4&8Gr#>h%R6P9Q9pb8goklxtF>h@CLPU}r1rT=0Fi`rm8}mYjy6*5yrH9DH;} zZn@zAABkm3+$Y2UR_$=uMJCiP*#Lz4KSg6Lz(w>jd)_RJpalOw!byaBTq~Hi-LCW= zTwnsXcp$S+8)V58RW32x(krzG#dLm2;QoBpd=CZWW%K<@0#bX=I*Pn(m-uyUF_Zp! z76|lBAyi2xx@{ztu@RMKSO9He)@wSLJu(LRLjO~ByQR68iq4L_PL-Dk23BCgaIS2O zyCMuCAHyz*roAu%pv;C886`W)$;TS|Zu&!!0pk)sA}zz4Lp$&pg(VFG#72NRaBfbF zrmjszL9YGJzYnCD46EwuY6XW?eI2Xn3e>`-x^eM!Nz=@ty0PhdOVg};Wt5s{ew3O= z&$ha8ad)U;a#qvKqAFI+ZIqg)m;Jx3wf}vd1X$G9?~%U$CQHPtxwUToFJyU?nkVn& z{vU@)YHscSj4}T$o22Ic7gm0V#ZXIC+QW5YtQ@L&=Z|(DWj^9_vrdvlG^o+m53Pn? z---}ffif-*=zIC|_m;?7#EB?9hY%Urn(^sihT~ZNyzjjWmwe~_*$A^x!Cte^y51tG z3z+3fmjWyOCk4)&IEl2NFkU(a*m3s+Gx2W0tk-K4=ckZrq-8uhf2d|OyQbV4A|Hk6aT=y6(iX2@5?Ri$GY?AQ%2Eli0NOhr zE>Rd5{URhon)mmvaX7qVgyRt;gHAJ4vrI(n6+it8BuHIl=p7DF0-m1rERII zyI-9%98({vPh!ZHAkdk}z2Wjl+W(DWZ^FX8sqDfbE9u$NcAFf_K$gc!;5vu4-6VSS zG|IgrXg+B-+s7V{hZdJ8~3R_xcZ3Eei!@8?5FmK?f&1?hm#WmZTqX!DCr~{KVxFj$?Odx$U>2+Z9-Br9(WWhytN@z@lbiVH`knbebA`~QZ>yyw)5w*mt>9l&)%)!4?HYq8>^%? z*`I(E)KR8Wa)8=1)C#rW8NNLtdbi>Ek|w0I^Gua`xq-9zq;?cg<&0IfVMHk`UnW1- zC!HqEQ`v~NL3;IaPIk}J)bRt}e4B>(;fnIJa{3jDUnxUi3H35s#w&?pWV5GM|h5=wD))ZgNC;aTeO32GyuyYZi7^8dPE%IWP;FK~!MO zD_WB0k7j?~kNYoZpW(uG29yn}F#|~rk1GDCjWJ>%@(PQ{ZOo^SKMF1;fF4aWXe&!F z%4Bybb;-^h2>*zMLg~m4oJvLZvU{%U?d9#zCxl}+PE(2Xr=ME)%xMHOEJtPcxQ(gC z$lL4Ezw@&6k*jHgKuDjTUF@GLZSKq6m&<+1Au}D4l9eMF(~iRW2U;4^Q{<=TJ`)Q` z37)`$Qqz2fNw97%*>gx3E<#$Vm#7w^87V?EMWAw%lPL+(L`*>GE5-nkWzLt2ojfcm z2H`T1mr4ur>aXD#K_=YK*gVfGUCyFDsE-*i713(PQ_?|jpQt>aj+sbpay}7}D&E(S zdT`RZ57ltbW_8o&D-8{tXAhNSDZbvi@0=As{S`m`WxurN-KoR6__nEkA17igy(uNT zkiCtc;!Q{W@ZOfTL~Q#7od1_ z0Rh}2+4m;z=xC$BgMxHAnbgMPJE3$R#8A`AC%|F2{N9+dkksgx0YN zceH7gsr>Ijy(!DVPg>ZSel@wBTp1o0wLV$Z``{MhR(|UfTMGj_Z+)WbGrDvpEY|7* z)Yoku?m^6ta?#y7*<3lD<&{HvD<(lj{U^2)pp)$+?HrbC*X&Pq5$ufKpt@%$!q;+j zrHsGN?#5Vymg_Y?lf5*P`axZ1R9=mjXWJzba6gm$-E2>}ZYKJ9mP)2QRs4k9BrTFBRB5i$SOs=|R)yT><@_K#~QXK;I6} z$SZH=kyz=HO?p6q6__5LnG`OumSwksc)nqheb6DJlENj!#>ML!L5#%p0PKqSi-w=P zE05M-+T1qoV?#FMYgwT(n-yH6r?2Oh$PirOTMS}_TB5fr^5i)~4$!ijyUnU8icHWh61f_GF- zJa?Q6kL@-0G%!*qEq&RZ=e@PlbKn|=UM&Xnt3;0kQ6kt7&Jy}cgsts7?e~%3pe*G+ z|J6|Q#yvbTYOAgHAMpC1`l2xH@O}J(Q|)Qynt+x%5i?hqLfy~#kqjxx$rbE9Mp2!A zvi290IO=wUSdYx_vR0(ysHLsrdOl7&1h%@`NlZu_E6g!zjFUiLk7Rs)`#?jRnul!H z-1jjyD>|Ubdg>;VBKC2VojDH;7dR`{A=e4aOM}1)iFE}BLpOkNQPH1N>L3ZUktZQ} z!^8~S?m@!{aK_((`+d7a^xArGkg0R+}8igG7*To)ya|ili7htl*BteJnoBD868=lrO$uWe!qu-B5FF zOe+hVR8giPMbF!*6MKd=C%8ZX(B2!_e1isLHv+sK+KIA%4p-VE3tTGMZy1GR_-p zG?d`lGq_HGg?#iExV=676v`7w*~M}8@}Ed@OO=PbVOn`Z&V)hy33o=FTYJ)&Wr;Fd zEwmTeJ~sM(2k3$5dR-oKJ`K6Y;cGukp9nZEki&yJJ3{qha1d=nfc5jvPKd7y$mPMK zbV9P}TOq-PBS&B<{?HdhG}0*P@7-=K9xESU8K7WlWI+>qyT1LI6*#~R-}71uzOE2S zo~-2#gY0)qpkwTc5Au4fc>`UjeWKy%^wk;TDOMi9>LbaHd%yj9VF!BbK(iBo86POH z#?sBAYbW~Hzc?lwqwRLpGSxCLwRBG1$MH-wqYOWtg7dB@5D2F@OJJ@fKRHtXo#2#@ zJVR#t7{_nTw^pU>u01Awjhw&8Rl$ts{YL%aS>Vgf$a8ZMeDy62b(Kaf(;6eX4AYUF zA8?DhhvJ5iH5wz(Po{4>2i2Q4aCWtX4d$uIL@FTa^_i76(i2BvM{*H&O^O4Ww4=2I zL#|xMdVgAa$BJ?pkd~5S2v-lZ({rRe-CoE^Z-ui~WhZJ_>1q z%5xFR9hpF>Xxa!{T8AS=GV0c(74k-*v9X?ZajK1>tROWuQM9Hu6jp{HtWk3dOqo7L zy&+a9+BYJI(eKo#VI`ylzF22Gt~P-Gd!jB35TUreE4pu%S+EP7ms4&b4a?Q>fIrva z$}Y+#OE1xTiju`pVWog>EXi1`jxS$u?u7A9Q;HBz!8}tCF>6B9f-r0}9Hip5z(Fb8 zX?bc5N}UJ;-!q;4!!?{m!rrMVUQeZa$sFgR*v#{uB+y=viXtbk?c#F@=PO$UC`JN8ic9#s@%iZcwO)({eM>L7osH+%=ZXl?}^jdzil*A(kYCS^B? zojJ_hqTJdVBTh6AZ9gxI=o4NKFgfQIPT+{IBx4{PO(ZV?aY7w4pPT9y^32oPRo~Vs z_pb21lIDsi`fODfPd({INx(MF(xg3bxf^nBpj1zo9>kBXygOV%@3pGX4$nJlu&B#h zZK_^&R_pNbRCzj#YmgtHEO*OT9KnqtJ+baFt-5dwlFX~o6H9A!m**>>Wk@j_{t<>$ z%qG+_ZVgD_(mz}HxsPWA76K4Ah*fRtzZXY$F~u~d(}RYNgPz)5wOEQc4WatIqApNn z76l8e{J8Ah+S0e*B+2Q36;D+V16!OG$eTs!pV^D3mg=!hWvzcI zZ&M@dQa;38R8e7g3Am16h%rlr9_N}zY75IMSKpPmsVl@5ni*I1$kp@Vp}q2cFOlEo z>^)Fby0TP6uTYE-{0?GM9+xT5&dizh2=Qg#rW1Gj>36*Z4H+hb`I7DZ0Z_ga-8MIV5ega z!+BBt{Ir7_T~fmD3(s~2rEETD5hEM$N(0AZ*nE?qOCrKBHs=x>daeZ+htXymfY&mzUe!*bP5>Jeg0qfStO7 zuT{cLm~Igo2aFfCAVMguF|QgwVq~(Hpc`Q#@l*`&*-5Id8ot7wiiD$)FjI9-GP9nm zSj@$&uhJATf|^|fI{>~Y-W28MZQo@i*9iT&dB5Us4ZaQq5D@%3V%HCsKl_j69%8_e zROAEyzZl}>vX%%1Lb#k1UWN==Ix7ZC!$ml2@&@88^)TFM!zbdzUvuB2pbZ z>_Hi|35F0)Hq&f~)F{o23zIliRN^Fkk(9wWuEW<$OkLMTeoH{Amz2gAEkkkyOkePW z|1nT=@c1#zr8s%>NyHt`^cztMi^@Q(<2IWkl?H&2YTb5XtU)g^4>#3=fB~AF!=_F~ zorvn7WNPhOc&BdtZ$r>61;CcdYlS18R>)$j$<%qmvygkMx}jU>l|8d1NPE>0o#XdF zA6|)eoM}A~Mp3x&Teb`^`tOgK8g*51e)-){eq0~_&+_--+}N0egn;kw#b)*tM&JCQ zfxq#{#q8JRYH|K}v;lD>E`fhfE@@R2UqwotWg#u=%nC1qhCI9KbVG`HeEo8G1hE2t zdM$o8gp}#l?`LO&4x?;RxLs#^r8vn?w==;rN~$9#lv1n%Zv zE{`({W8yn}9uztw>wKe*MT9feZ?E?(d3&k8v5sq67g&`v{#~4N6#WLcU3K_T7R+y%i`qEv?3ybgO@D{~Eg~(vm0? zGk2EysHz^<{-c8*QK^tgiKvXAGP3|h%eu4dirg_ZdjD3N8E>o)_*jlPaGD9Ge5=|D zUA33G+X?xmrv9nJ`;4W{(eV8Jy@^)&Bds8+_-{SwE0wO5>i+eJBobwwfE>5eDxT2k zaQOO<5)Oko#Mk@DMc!R$4 z;?~^eRudg-u8G;vZ*EFtwNNA&LzI%ENw2<7{4JOO!f4R`q!dn;ksOrt%5j;mo_2r# zHp8}RMq-e4Wueu8V}5>fz_M;aY8BOPd_I^Zpv=BMK9v@lJxWwoa=;$bb7E5Xrj;C; zo??})<{7@z{JDLVw)3S}D@mlOC7rf|&>pNFXBsuf$at?h8G>^*?|ToMRPZf|cNIIL zQYx0sgOAlkD-;Fx`Z(%$izLOz$MGASvO~02NyXCzTDRt@WS}pggkpK+zj2Yi+Zy#ZEM#??G=e4cosU?6x6PrRxMhU zu|<^6;xKhsRv6tsN6Uwff8#LfLa}Pay{@PtI&f@*y$iRdQ8Fl5W{p8M?b^b!VekL8 znH*F}9nb*eQ7tn;)G3oyjy;JFVWv(x==FX^p5YJ&38lO0kM5VIwY$}M9-%+`(eoWS z)F1Wnw9$AYJ;+M5&N;x#vRRqB8fyqlix;RgPNaAy&4~MZkQuybSV~DtXfqvL3;Q|&MRnMwIZ{h=@&dWDkX232P81`Sa6=u0w@f@+ zJvJ0L#F~|%stUGRcQLCY@NbxjaA}GQ)wqui<5lt9>_|*fs>e~hp{XIfv5I2Qep7ii zm~F z_&N(IHYq?AE!El+)QWqkbyQzwBx=p2pEaqKqYrH!l`s`Aq&%}nrtFH7IlB6E(9b`C*!*Z4QP_ZcPc5Fm;1UqMv#UvYunD`gyCGPZJk9O{)r~e(H4p<(eX zvxKdnX{c6;qFO#q2|6=PttJhpf|V!6nh7CokPde=q#3t~O7_29m79UG$t^Em+4cz~ zY!BWPT@*^Dkgi)-HS8LfT@qbRcC6I%w!~&r0-X9MhG=V^!~cBDxQju$lfW|wllVs zx8#Ci?chKrXd#*Lq#Dx^+2^^H?)wBlm2oEOy9?13c7D@_W0xh|fa*IMcUGeZqbADQ zLf;R+XG1Ad-P5wU*Q+8-?d+s==C-V=tNz7tTBcJILv2D(UL=S8Zky5k0#lLp$=B#c z_>Wb!wKhtRk=b<;MmlR(N3j~bI*s-t8G{GyaRMQ!Sm-fs=u>@CVgpYXf0n8&4zdfK zr^!(h+A&y$L_=I76#9g;1^H9#LSK8Y`^QRWwYXZJC|i+PSKdWB^2y7tY!)Ldl*mmy zV0iplCZ|cNEfuY+*6fb)ri0)v5a~3SONzpSVB;|D2<}RanviT#JGtZ*Ekd;MT!w9Z zbmJRT#Juy=2Bs&gR4n_Fo+%Kao?GOwW(+K}dOP9i3&p4tO$`Tp zoc@ZW4h^|C>HXZ2hc~OZjwXvqxiJbs7~+G@Ry22tM<~r^CbV%e=+6tM{~qD~q!3{! z$3Pc+K)}THxM|nn4CgfY6HXjP##|WsNUa_eJ8dIAZWy7Qf5f;JNx=vg%^5risD^F; zS+VkI{FM`65vnf~EEx_b*Z7W|5gIM)c0Z5rqhjOoteE?wB96bZIR4_l(ra}=!u3h% z`uXO+_UnHJev&G{g2b_b1i^B@>yr|muc`#W@`A*2&rh}#iUh%O-I@ei&j`0yZF6mQ zT7xSXxLY1peuAzmvtPnik(dO*wIEEJ;iGQMY56l7oH=ahq7rTV zQy6L(2Fs;Dj+8*^H+j|r!BC9ALP$R4Gg;QH@1oCS2M){j>-2-86w$iPZRSz2s*3E3 zp0kgS=;6snFBQL-EOh3@;>=y-${@60?Cy?wH<&qUTF5}$zsS2BARm%ruqiq7!>}D;vS#ur%-|1g_1_qzZ>N3jv-<7) zQ1i(}mfkO7?n)*{bG_7a-lt1<4F+C(=}@ym-K|Mr@qh!l4@N)8uq@(}=Se{g*E!*) zFm)30Mzpx1Xdg+B2e4wuz$Gc-WK^NWNGT8}C)-LZO|i9PcH4niP_=g2zdFHi;!kz` zs`@sXZjZBXb+aiN%#--8x?m zPTF*~FVx?Nh2NRK`}g-cbhO4IHts|aOarQwxuuapN;K6EF`3BI;TTgSuUvlA;21Bl zDhtYKG&DEfSs}xY4t`(EDy`Y?VO>S$ES(J_AW-CAc*7v3@hi&^CUI5SS(foUZ?avu z=Zdg8fACBDvVxerY^S*WT`;TvPNbC6lG_O7X-2G;%T6C84LpL`Rc19IIVW9iEv7&m z1sClvOf&~n+{AC#!OR>5q{lxF0}WOa4TxM}H|ypB{%fNn^iD%sWSS!PP?^9W(E>FU zRGuXPo(b!WWT3VXj*uymix6VZ8f&ri#v~76P^v8KUHxrjL^V_tw!&Ol@{v_9NmhyY z_1NwVxIQ95PqL~QhG@L?`N92B6(^L%)B9%_rx`0&g99Xe$%Zg^6(6im%|_Y98EA1a zmC?(l3C8qmSwVk|GDvHsd>}f9MB&dqnYc95a$x~FacE%Au$y8o0&XCA~pk`0~5qF;+JK{rgGNo zT2=!M2n-;#x^f5Nl<25`ltm7a#qfH~8vhZ%o`YrZvyeiBKK?#L zDL<*gtjkNt+<6m~*HV*-g7*42f5+R?|2$k|-;=#Ynd1-;s3Su2f_aRD?Jx^Vk-Q1u z93t>Ivx3}|o5pNc9D#}br9krwq>!x+RE15rd?lsjqF32Nyiv@9(1H&lFzEGlg%ip& z1LK;*O#ah1ghN>Go>D;--bRRF9FDNBeh`JKy9^6&X2R7>-;rDn^Jc%*BmUS8ATFC4 zw>e$F=c5dHFx!UtUW301IF(|TWN z(8*u7AADWJq8g${MZ50TO*l?TTM=cnYN15&HRtS9(wT2c)-~dF#B>ZzOS7J~EON=x zwKvAHW+av$DIAf!qfj#aVy9^XL_n0E`DibAjeL4nyL1J_=9#T) zy%U`9KEx1s12YaBJECL_DQF2Dp-D&`gIYYkHU3?2m^-^P+-|ztvQb1TUS09g(N=Wo zy+s`%6XPu7w-uI`&yuoX~3=7VoI66pJTrL^QsI-?33icn*hAoMfQ8}Kbq*j}ela$I&FMRM?m0xnd!P=g= zN6O5}F10b#nP*|DI=1^Ey_UWe-A#OAXe2_POZ6yFEv#}8j3S(WYDfT?KxV(4YsEFY zfT(HPt=T>Ok+I&{l5={9ZfMZqw!x^toOuTk&)dXglHefBx`MJo+nm%%D^4+|VoFR; zvOXV-mmw&{j?)krEhRVv#5SKLE@h^T+dKKfYu)A!|n%e!51j^}}ws}wu9hk)?f|M%GcT?2Uq zm2Zin`a$VR0-JM_N@Tv3m7+JpCnMv%dN;7L1V(F1{mAn#YlVJ*PYB_Z|avh_17GJrIX@6r8J9~#A_0i!; zJA{341b&}pY1w+PiILl@RfsP0CTcTNX@YO+PZVYfs!D`<@1=~V^BCa_k2#VHSG+#N#OM`T1qrEbb^a$IR(Z&X6G8~LR zaoOh~l0?)_$u7VI#q;PuahYP_*~SnfYVuI(uO&3yjlKoIt=VN~(ZjS7rXxnmmP8CY z(qX-6#5Qb9+cN4C4)Ae%d<`g?9I>Ey1xa3=8^*Qe{ z0Y(qq2jz69P`+Gr1>fHLE1N9VQ>WPo!Yk9H!~k19#Vbz#vOPekA!|016L*yL&qflK z#{D)_x7-qKlBAsZ$lLKKh#beZA{I@WlC`-90*TZ*xv+{r6udc$&AN@Z+Z)f>w_lE4}etmu>e(t*-ADfy8eqKH< zUp4A}X6FRH3A3)@exA>-AYLwRBSmNDe!mz7%(sWMSNQ(*1+G5~@;>430OAX%7HVun zV+ZaW3MAaZ_^yalk1@Dvr7lL98#u;XKWMbd$>SXDOR;l*=2NXYxcLH+3(9V z>2!kZFGE%olRjG=27Ru_vsfcb0Ee}-idQE`AH?0L67F8nXa$0`N@R>rVlVjq>#E|9 zrzsK%UXERUo($2z3-wQ_xVFSWFSHvWdb1cHRG^lRMoXH65KF8OgJd+xtT zuV0Qbdw#5c;}(U<-DShjWbdirBL+iFh)=;Q7ZajxwS&H(VJHi=)nj^4|Lf>+ccQc* zIGiRpXL0A@&lON!(v*21V*PndduG1OGej}*ZdLkEn5LP^3qp^ zq_fWvUKUqj{H3C!3B!tPZLD*(60BpG@l@+I?rp7e)+-M6a$}Pqzn`B`1{YwO_y>&# zCkaR%v9WM68kEZI`u_njK+eB995q5lr5?qyrsDPPg%maWIO@dja{cS>Lmj?v2@+Xi zE?US3xmO8r$si+6?;sE)6svJcfUAOP+(VZ{t53eoG4`YLC(tIC^|y_bppg zQ+3mx=xkofygk65*r9>1*vI-_rZPCbt}1@Z3$|vjR7MMsAktk$7b6s4Rw`L2eW+=n zDJ-a4cP!YI)|3l`E-5Wxg+5;`zm>GoGhqFGmQofbr4yFBX_ra$74jc~-Tgzq zNs(1S*YRL;W7g`IIcr_YV}la&Cu$L#P+;<{L*3u_&&jiMSld()M?Y+o{Nfz8{ELgL ze_a2^^*=6NzP@}z-pHESQkzcN$eevQBopb&IR+L|Ex>LCG5R8xcPi-0Ku)43gawcc zGFnn7nHW*<_F~=r2jO7K^Cx*FOKh+We(+3@!Ct=X6%MqyQZV^w%T>) z)6`*HM5ZI`EN69iIzm3UR5d!EmZyD@=Pd7xPt0X>KvK*LOWt!plB>7Zm;d*aHxsic zAI>0|o@t<0=A?0bXRr&7BYXcOX?L=u2j;$OL>IxWr9rkbXA5$b6ol1%(igBRM)c#N z=Ag28@yg2v~wO$)IlTw7E`>mF!pP%M$MX`G)W&M@I z(@0)9-=jp@bHjp|M1l4kY&&{Wg@><&@^BdldZ@;?+s`(DvF%nh;zE60$tHLL?Fm?4 zU%u4qL`$Hc5Q)0l0dmeBL}tvlih}aDdWZD(L>nhZUeg_+g_0>Usd4l6RWYOg>7SRf z-Xtr2#Mr;~Y0kW8%r-cNi%Fzw@Try+%UI4t7L@_rk_Zcf)0fM`1{}h&dLxD2n8^$4 z(ngxP;{V>J1Sh%CctSvr1Z=8s4mJvGF)L7~aJK%|ELovPkQaq5tj?l=&3sGOjL_Px z&{`5ER8yIW@@-6}iQitn43d0G&U72kVQoP@_FAHSrmyR*Ecqy^AZ!*QD`Y$~A|z2w zw($$dTf?A9tBM6%_4Zxj-U^cJTYDz#t=P8jliu4~55q0LRjiuCEWdqVX6cJ`dX0Cv z4=&a%t7%T_@J>Mn>k9o(nC_@5I>Kf{(;cK4XR{-$G@^#b*z7OQN)`Kti{>LHr%W@0 zT)%=xNoGd~W)>q+b8F2=492J!{JrB7N-?Dh$i=+CK7P2}v|>ytjLyM2Na^YiIbl95WWER);vs*+mOoOA}5 zM_UIP;zmpiJ_g>|B}6MP360jsRBa~O|B)!u04L}lCiV>-pPQy?eh4=;k$v`AWY~=J zfb3Idv;f@k_+*m)g!Uh9lk?DX2H&Mbs1uZ!DVQ#-!BEJYd9G88`+bi|Ui!W$MVnWs zPc!Sa%sbR;X~01Za6ptKg#lm!T<#^UH#1VP0y7rPj=Vvg?i3cYd1;icUCnmTY+!r~ zX#ldq3RxgmY7KPruthdX)HbreVZIfx z40XHbS_SXk7bWYB<lV@!r*s8^DS2@7IvYpJGR_Nc|*>ErE%tpuhkQ?*yp+jabRpN3K!G_ny#o_ zY7R6qbG?9xz#0;9Zn1L|y2F5VBlJ{1*h#ZLRq@=~=ybg?3Th8$H`?*RF+v z(8`#a1}^&7_BoO>j1grJXGHg6)Kbm^YxO3l0NGSNa z0{d`5rWPUZ%8_LzBW09mMumfGuBA1(m;ing0IPLwTG6dOY!op=yku(*fsUi3-HCDP zr3TDU8ss(E^7RIVO+o>2iCE`nA?f%izz&ubxuexqpLjIefcKROk~bC9e5e`CcXL(K zS_|b2vz5lzJA}4m$0|FAwT2w4>>R7?L`U>kWoPWJW0jp_m7V?jj#YNN>sVz6u49#* zq3c*>XS{{SDm%w2JA2o$%FeOMPUt#T**R9(G1sxm&aukQ_&vueJ0YUKTJY#&m7V?T zSY_u}WoPd?R@n($zdrHiSY>DEI#$^^R@pgL**R9(Iab*@R@pgL*%`Pe7}u2ZQH;A3 zs1OtU-)6j$WwTlZ)`(_^@YcOxg%$;@??Yns4A4l%=Nj6m;zu8L0-N6 z%B_@f&EH~}1jjcN!w7A9E5==eBZEAYm|?~`E2IiTHsC;H8!7~|$``L8+-%91AhKk} zp+zZbpaaQp?1jyq6w~?ae8&y|~ZgHS1wxt|jL1{ldRw@V4uepB9}2EOL;<%UY2j{6O0Pd^O3rGTM=v5n5a32NQ3{9j1Of?gS4iYz^M(zs5sPYyq zU}+yyUcuPjP6jNOqEikl0gd(aRCQ$CEw;zWYj+i|YX%bxI9-7m=Axg4d)kfhBCjp+ zmZ&}>L>{1urkiEO*@~<#!0j6O(SXYh6Xx&PYkiN!O zWDS8-M!^34(|`L{vfSy@b(EqO8f#h8!VO}<#9Gf0uo}RR?m$Ol1EBOrcmSS#akK1- z4o`A+`^1~Kc_G(}f!a?8g9Q&P(R8eMixosR@(%Id23A-+L$H)(Q=rxOE4{K|#2Ur+ z^K=IHQ)omj=^pnnv2?bH(!`=0YoX?0gyeRDED>X;Ek`D{5q|B~1L#!ajIi*J(>QS)_db{y((bIL;Mo z#fsGoGbEg*b_$2kso`#rlClp@ckuagD~o2!Ub0)BT70hagtf91TmcHwvXI#g$<2>- z5a*>Wp7y|1E&UdDLO-^&EV)=?ynIFPu(?6~4ru804!O$@R@D*-hmP`rezgzTVK3n` z24+vfpdxJshwCGrF?x((hdx00#+0CzP7g((j2a~+s}xt7m462_4297m-E2*E_wPJzP1K*$~45n9Pck4+GVkPC$#M~E?f_)ms_w; z_)05*tS!|Q8TT^~iL_Md<({4oA!sLbH=fw5n&mxrKW=ke%r1u-HL5fLoKZcW%e;E8pUrPf?j%rmX;+XdLDR*u%BDuN!4=d=Yk787QMG{>R0$r@!N`zq z6sxQ6-W3X;lCx6getpfo2Zk7pB1%dCGs0|oX!Cg+b|MHZ%8d;?0b#BUIv~#$UkpVA zppWm_;)~y#k=qMafgOBZ)0(ZC;+oa3yOT9QvlFGV_j@=-ArzS)WX`Tx0q6VAaR9x9 zZH@gzQtUtZU;qHFd7c5r0PzukYL-TKN9Bnp4mCkS$i z!K1K%6V!r$4p9Dz7Yu#Wg}j5H(HtDMTvoguz#9mgY;M?yR=St|b;EXT4s%v9lgOzj z9G0`Su`I{I(V<#{efoxtfFnf2sc|L07kcDH(i)|to{Xy^Z&`KA+1(Q(!Ok@kn&X73 zCy>qeQe27|;FQit!}C86`yC_WBCTsX zJ~r1Nbnk50O5`CpE5JP!Otua33xV#2Edt|`?_9rO%BJka5pQWZH&Spdw>*n);jab( ziDTOsJg41kZBWn6x*FA|>|XEGEC)-mF{gp2hZa(-!BYY9>A?Gkgu1b2#zD3ME2yNp zMUer5meY=$le4mrJ1_xoWvBpj6*br|?4#aEL+6mxPZw;sc#F+`gcR9g2i9!db<+gZ zQs%D>^jf5ICs*yQKKDkI3d4W2LP)V+W0n|$9d6CB#$VtMSt<3-{~&;?wnf0f@;_qh ztr#SLV!&#P7G=VEb0(gX-;xg}v@920Zl+n&hap%6=+(@IUwt@1O=}Ly#fOu{2stde zpU;=OB%);{x0=H=YK~A?GC*7FBUEH5&Fy1q_@isd;A^_wdO*Uu?O9#)`e4-9wv_SU zJKJS6J^}mBh0k$sX$~I%sR9^DF&k|I-x3mP#I2x~Gg8D0_ zWK(4Z+ILHL7JzcOYfTKbl!b}f(`Wj4bVm1Q)h)NQF6=LA7D=cZpo;uZ(?6-*&^F)7+Eu6AQ8dICI~GxU#mE+ z4d<0EMdgcQqeko%BwR}| zWCEVNr#ZkluQ;m|DhdY(_ks0N%Bmg)&oyV!b8^R)VaW&ayY!I4{&b1Kph=6IthJm0 z*yO`Lb3e)S1B5EY*P<1-uH=B}BlvnNsOYm^ zz519HP0sSKD!F|nRRh}-6d z`48=w?`3tPh;rn^tLp1S6RQaCdFArSi9#d zITUMb;1Ix$Ob3t2vpnhWWm1A}-4_DBKK$;}{QS>nXFoih z|J&bxclKccAAWoO=krhYkKdi2pPl{ix7T04zk2m|e*WnP(QI$s z4sN>H6Oe^W!C~H(G4b*AV`&|=)RG9_mIcUnn8R*mB5=Db+EyEPG5eS?mYXJw9V?&H z<_i2+v6_kDTq|ew{^IxhcFV3R4;68(iNyExstpCW6G#(-aDbJpLXa|Ck<%}Jzd6OQ zte*T)v2|0>N>6o#Fev0lvst$4llrv%wE2{Oy8E;{|8aoy*_Iatw;`1F-z^tS4M}26 zrASV9irg`FqsWfYN)2SOc)R}TC%akCckRYKzwv{`czq&*%I5JHU-9 z+0^W+Vk`bJqUOBUL#Xd2CEVVbsfw++g2G^clD(lKFNUiKtS~E|D+{-8RJw03jfvK{ za-|@YsAf4Y=}^3Gs7t=-Z#)ok*q*6eD++|6(+mbY35XtZ%S_IMU-);(_Mz1R2PEH` zlBwE@XE0HTfFd%UzZVwKVU@pf3p+)@>py~+Y}PYU7+(aAUPp-?<5}-FJ!MyGco%P8 zj^%5lt2`Hh=3ukkFPfi^it2_B8ZbUJBb3~*-OPj(5gQs~RffWp6J&#?A$zqR_WNj8 zqgKy>;kaQthiw4kqQ7uC^KK}xsC5kC@}7zfaif;pd^gM$gUAh7D!LD+%GBMLLTz{% z0~Ms;-I6Oi`mQWIlE?LTDQ4u2to1*yK5`YwRMT_$QZn^M)^PW|2M3#$JTxX!!x%WU zKIg@A*lu3Ns~5W==89YjA}c!vF7rki@;-nhAg+Y-=q^pTrAHr!Rlma@IsaARA@4K; zn_$id_yr>Pp#W2EOFL3MBD7}fJu7SX+Old5VWJ__wWRtjlgglm3R1-F0OPD??uPww z>;!NieDjM$E_ts;e!j&>rvF#7XIMo4ALg(v$i=w*RA2rRCPU08uIXxN|DO%NrG<%$ zp@J|J;75qc)!u12BX=9A&{8>CP#TB2M79lf~Wv zJEp*fkKIFM{b~LVErcb?UVEo|>%~J4MlYnpp@0lam*}GhJ^@zYOjbD;>!U<_8ldvqeS&!5|v_E1ChB>)a+v{-~YOMH`&M{R|Fu_Ax+*= zBebq7z62xYaOOrPx7#kxGpn_Dr3_Q))gyHX5?271?>Azq2)0sRSF$Nnf)y^%YEKKI zn$lY13$oVFn{Wd>yJ2;XR;uuDZ6S8Ql=*^O%dNxy(eq<0m(bb2HA_~okt}zhxPc-s zuz>G0VSNls<9Dz`!Q!ZY_G>_l@qrJ5Jj0bM?;sZZ>&ureAVfd=)in$o`2z1!tX}h+ zEO`L^R3H0ig54p6o`VDE;vU&|xQ3b+jbRFXiSI92wdH7TUveYEzI}IUkXoy|d386R z&*!GFF_cpn^Wyj;jsVxPHt9aS5!OtgeX$zHr=?cE<}?U?EiFu8VHqF&DRFOGbl-PTw$oIFAjr4bCuY z#uU?AGQCjQ$1;GeZelL3^p^92ZeX5)YwI1n)0?HU7ehc?nit5bwnIghRI$8m`dM$4 zoI}(Iyl_Kr$EH?D!d5G-ok~GixnWyY(W2WZgX#U;M3&A>81QyD2$vW+8AP?5S61$J zCm16qZ^b()>(^Ytv3UabpIiX$&dHc4wukKGo@m2l0&9MR>ex>3*Ze`@qE#av%ZRAuQ0J_($S(ur%IC*aHcR!veRfR)=qw z0w_j$%}^~CYb~vkql>*!tU~<|5FhsrAc!Bg0PFx)MrdtU(%2~`2@?vDx_XdMrdi)! zz9dh{8KC6B5?N7RV1_LnEwR>vPFq>R3wNw)jnb5u5E6NmY5#aU{~w z+I2rpIkp=@GWcZPW(TYjUI4_<_db zQh^`4Fj5G~RCIFn0V*o;!6XBH^;_mL4!!v~9-{XZEN5p zTj9Rviky67$N0qD^XBdQaMx{8Kj3d-y5z150AN2X3p>G^h+B_V>^b=$h#MX?#+OD0D#IW=&5nG1`7&f%wsyiZzG{WTb95=8D`CsFOEey;c<$o>n7PArX?Q-a z366`w9m!ot&w=R1v|s44Q;EXvWwldTJ?2{OdIRxJ@B(@vNiHlsx3nDaP+(d? zx&d@w@ZdDUnAtrlXMKsu^2;$_5tIpVX0C!ZBHJX3PyzMsVJ356ES=AUk{ebDz}T^x`!xMzTeSyI#NIN#$# z_JqltL;YcFIaYw$uO@b?5)E#CA&*7G5IAOi2{4ejA#mSl3%p>I=44Y#q4n28m8tbs z(H9j1JULoimn=&Zoxgs4ebIdo_>#$CXYf0rx$#GaWM{VANO?ofY;xe5*PCX!$mG_` z6Pzo)R!__to$FSI%KvatIPi*2Ra)Dy%(Wbb6Ex{j|pwZwxsssu$L=NLQHG>IX4 z>RWOziUOJZ6;y3$xm18M=<^$T%i5ecC-!w(kG$awOE-)QZKTaa*CP&;DcSOmx@A5+ z&t{_;m5^fY{T!NC_u%=0T(;`POpmJn2Cr7XfFItl`c77%(ycHWzAfD1{Y6$yxZs;} zQBBwe;Hol@Xg+Mura{+Lphx4X%*$SQSl5LWHX4PUTSFd&l%TWrI_zYN#0npBx;>H= zH1MMV5WUDU*@#i#(oR5O|A8T9*}ZF^#m1uQtuKRWg}JZ*fi%*O!BH^y{;|fJDgc&4 zD+4#Sk$0{_04AG{8g9a3LCzIj7FYuII#hJA8;s?HahhUuR#T55 zVrnV5U{tYS?9Dg>f!L$bigMjnzcTk?Va{uV5H20Av{C-NVmJZ$)stTNfY(jlvWooZ z!mNITP5I+(TVwe>+m;2b*#OKfUK61d%yM{QN)A8yjg8709@n_ExMevXsx37~QSk`>GehE>$cnsXTWI0^<}5O0Buxra{3iw=9ph6Mgo;T8i%Eow=`R%5Jj;e_ zS<}LUSOMRST`++Ou-dgTO>#x-{&s<};2PsUF~oBMA7G(EK0Z)nGl0_4RJSm#ZjrqA z0Em#=Nn!2@2~D^&T;uMnncPR1!NYgS>O02pc)7OY6xuBAum)AkW)P@m3yBKYdIP_zFCj`c~QSrvrEn%*0QEkn4hgMdY8L;h=fwRX?0D%B^hm!c+iy7ey zwy=M}tzjboRD*(EuMv*dRc!@k2F>@S{xqsT{z-RPKlktRKbdfO`0Gbw0{gM|EgozE zqR<9cXa7fwd5OcPel)pXpumF1A|CEato5^E0LoL_iE{d_mevJi$5>mUkTntx&_tLr zJx*s39QToK%Yw~FEhQ^j30P2pq}@VGeih@;i29%axYeLwfeLWkzl|6`D(yh0JbPZpv$7nKAo@ ztM6Wf*S=AW&_E3d7g}+c+vFuZfLy@=TYKy|v_{>S&D^ryP>ce?sw1W@k}zA$fv<*z zX4`Ac0No!j(P}~7$ZAWA!eHPUu{~_~S~4P7#+0JfZUzP5xhu|O(7dq_5*1KF9Y7FF zA*ir0PRX*8H%ySMXjJVX@wOS66{Gna^KrnQW-{DDVvyxcTbjd)`CD6V)s_kQ?ggQ% znnA@_J>*L^>eeG@bj2bt9|Yj*Y58@AW;J8^7rY3>O{93>h%-EeGnuQ)Hi;PN(6KnUR;Aq(*V#&W8g)*2fA69LamN_fjEQ!48R>)8_{ zVow-H&^U~LtSy}YW1s~eRWnh4*PiX+CgOp`M9nMb+8_Wp4a3_AT*BRfj!?vS30Aks zlI9%Jtj$OoMsaf9j_A~wtkc0Rnu$}=)GW{^X1LpFiyU5@X>Rgb=J8wm(y;evn`Dh? zzzVcVydg@Jkgcm6Y|a+L3pe!ka2d07Lun<+-(Z90@Vfc4aoxvW319|zB|;kbH-~@? z?4>ynih@(;rhT$?jQSF-v4SRajd9Ijo|E5pL=+>Y3RfRo5McV7PdlJ#v;NCb-LGZxK$>UJ48SGTHj$vDf4hG6ZwWLh^#<%x_s{)Z%LEQod z>y!tH9-a_#-y9AotXp#E(^_H7!Yg@d% z!L`Kxs^p7*A+V?j#KU_QITTCt~C9`fupw zHYZZ;?wr4I3lqF#ZH!~Z^n0-NG0DPSUv=ZLH@wQ{C4~s6UCGqUZ+Rj~hkc!N|D@B0 zK)m(=GYkV}4bH|V;VPZLj3sx@&g*0JE*3r4Ui)!$J+J0x*SlCWTzk!7mJ@CZgwORG z85Q*LngVRozdHQyaG6Il&u+nZBcO~_OX^>)dQvINrm#8Q4hyB;KtpF3WaGfT(hqll z&QS<%4=T3N;DL*@Egdag=3nmhE#o3*0#JJ@+i!5NjVKohuP}xDtn}cU1Uwh66zY zNYIE3!}593k}&<;6WGCaVBkdf3Zqn#R7qkqeYY@X7(G7_hh@qXd^H;L?O#ip-Q3YC zS5U!$)_loz7iTC-`GSu)J*|`CeC)+VU-I+1OU}Hp*^>xUbP2CymbE&y-R~AV)lH-m#+4|K5+=s>M*hv8E@~Bd~_N1XM`8&hko@r7^{C&-?whmGlAP z%woDGPME1E4mfgzRh6vpXzVvLO#t?^s_@ifoJ0DM4o|iH6%25CPF+2Sl&x^~7gp1A z@~y&wb?1Wfw2o-o38Z~}+}5H-g?mBHE{byFz1Pz`Qz%0d<*GLxoi@LyBvt_jq7U2r zquzSdTaS8c_xi7_w|bJzfn9V`QP;@kb@sq5>LnV297`uLQi8TXPYa^J@9yYsAutz! zi?y|azq4=sx}w35djD!m1i5HEv+Hdg>B%j|95aA1rQXx0%J{bR?^(vmI?=FB{N=MC z*9=vRZg)v8v&L0QOQfp#aL#JV3)P-^?}ty>M3UiiZK@HR;Bb0p+kZ=|PoQ3M|LWRo z{BnOT?Aed%jMb#ZB{Svtkz7Vgn(=yP>SLrT=pcpEv~E-@ zK7#B4PWT6O(z+RcXelvNk+y$LL1*84HXIt>y;1|kho4G5f63<=(c8Taitp`TlPln+ zp2Guma(}UkxczIAfj(9-w|^b?!hemukW``fKD%M8Vs9$Vk-EOa6?{{r^W-|eX%Pwp z7WK5czlmhK)q#IiEM>yYYP2#~|LUw>SBMD*W zjK3-_w@(u`(rSnWyPQx{EOBKGR~{RzF_uU%P!gqoB?r-wiIvI{Ns@j=vry%-befS0 zX9<fY?El#@hWcPpnT&b6RNsBOf&bYyiFn-t_YV} z^=*&p6rOMTcgVV&5=?EcbaNVBp(=9w_eM&U^{tI+kY`{;U;XK-rOx@6t1 zQ`;_5#V(O(q}JVFRl7veX|?ZMmAhmmCAk(}BE_jCY5Uh>X%zPhIW(5Y7X?MPz*Gx% z|M>xEoP^Q2_NcUu9t)-U&?eSQza}LCX2O<6+83^*;K2hrO32G|OcAFLei8=oyCITZ z#y(eOj1$Oqacdw_;`);06~BdK={Dg*Rwi|Hn&-mhTYJ?R_n>&Iim6hj23ihw-$)D_ zKzy@dTQw=0s+20N%~b5|9x&FJTqB!WhuNKJ`v9f^67+1E=S3Gz#<)0I?2HuWvm99g zJsffq9+%uqOus!n#kda73#8QvAC@w0wRXA*3Bw4u0(Ho`aG!~)>S)p=f{83;Bg`7k zqbTV13Xi1jx6P$?FTM@oNpyy_O~b+>RyI@G$ornpY{jZ7l?^O+!`!JYzQt^nbsv5p50Dz#2d(^N?7D4WN}7Fth-E_PyIY@$l0tH?lO)}^ z(mIFP82vdB?RxZvWHg=5w7fKn+9DWO5XK%}(p4q5tlqHE7zYnf$=IzKRh+4s+|iu{ zr=zYAD;Rk#67&ekZrFdD-hr`#mSWEUO$#a3isxfQ9qfifBIzxqgRA(^l7EM|PpNQ% z7gv|&3CCMN%*5|lPE@vGTZ%eLkRvz)DZ(+x61AuAh1JkA$ghHZX3bt-4l>K^zkAq z^(1GvtazgMdQPis!)uo5bL0sv`5ZbZz~H;s<|)xC37jAt+S;908_!C+60w^k8xa2P z)%E+}q#9uWn3{<)Y)#?q#Ok5vVuc~Xp=ILGn3%}1-X$JGT7)TrZc&(45rs>i+k)0HVz57VgI4GZQ_2WTiQtJRqC!;e3bDmG~;P?iJ zO-;3}8f9~MjL?RwUwSH37ir6# z@iRh;dLx_l29t|ZgOj(^q^uEY&qJ^h!BL25N0>p^idVGMT)E4}u%6$ItRB*0(>Kq7 zS{oSFL?Iy+PV6zD#6CwDu_RCUL|Z}3`Cl;IC+-_W`XUpkEP+dw^pP1o$-->X+TyH;`=aHX&_cH?&9GbO#!fqO&L;r4DhPQ0Iqrj(5i`mRi4}*)kEW8 zRri{-aiFSw7^)=Z8Uv}Cf`*X}Q04FgCh15I2TpZxRM1h+I_gJP+Gzl^DS>DR)@cy1DS_q)q|>8<5dN38D3z&(4@m;aGB}AGLu1NK5Iak zdxOb52B^|!3#c>+_bwGk>A1#!U2FWIz>+2bC0Pih6u~A5B*}w6N}w4BNE(MeN+3TB zjx-K?ltA*cWeusN5tG1>4gf-mn}er=Af+Gt)4`9X0w1|yNLVCs(4*8W_GDO#DKJJJ zJW&E=5`{3@gC3`D0W;lHxT47z7gJ%1QXz^uutUim9)%~` zha5_#`7qE#lLvMhBvCRP(L~HH5`!Z0FgVheZ4#5u%LJB2Q zychJ)MA)GNAcv;G?@WOjN`)Dk1Tk~~LH81;9$K0{JeX=42+(8zpborGnlP!H$q+r!0Yw0xDeyee-t&>TrBP^}*!BnDo@uZ= z(P^)F@%cjXJRTg+!9h2ZPNFyj&jIi|aW!ia^o|AAiLtT!!0t>Mw29|+8q`h#Z@D+j z&Vyxn!Uj&y^DHZt=`-wkFX~$`EgES4|GA}6(yVT1VaQ^hWV#E#ofZV~+0X~&Q=fY_ zr4|xkx-F0f^847#9HRDiX7hWDQ}Pbnie-%7GBq7q5BY|XQbG8HgR$Eq<5NnGvO>pF z-5%2pmq%d^oc__oe_;G?8cEYkEfp24Hqn7TjKPMAywKuSOvOA5_u*u8#u3Uxr3GsY z#o8(UlS#pc@J@cqgx*pM@;7GUd1J?@g!nI2Q<2juCx3H$&N>-(qNX3MRX54-s5SOB zZTDL-FDtg<9}z!AJtuRk2#`6RZYG8$-v5?~EXZ%ki$>Mb3nD@bXQS9*E=AW$7&j+V zeK=tssiwEcD*%U5{3mnNq$$$e0i4Xsf(o`6Xy;dy7y7VLC9a22_6 zbf^ zr>5GSZrd`03Duj%2+j8u6$+qU;C8Le3X{0hZ!M;u8MYPMOMFGx$C9GNO+g^j=gC&f zV=lUI?n|Z_jrD)YyzH;Ry8I2p=!g>({D&j^o(#&+y`hL53>f=dDFvCv8q^vFs|5t^PH^+Ave zpDmiwOh#_jn~)5jR5K2`9UN_A9HQlQwq1`0_ApHzV#^q0rc80ULy3 zBJ^6sy(u0CA@i=<0Am1N8}Q7kQi~hQA{uQiy`#W!mX1_6>viYqyB9J07j)Kwr-2RG z2>7=kuzHc8O394Vcr+Z$m7RPrc^k)mQEsT0MeIyiV0VgJ|ALX~)}FNpOOy9?T&*iA znu3PJ%e`Ix2zqH!5bCr5)JegopKajjy@F07N~a=qMY9{#F{?*)IZMUZxxag1?1y?2 za|pV`jeJx&k1FRws+?_JODQY8{vfk{8x1X3GE&fLZS!M1W2HNCOv0W~MVRHpv9;jK z6b?}jMQyvd{@KWy&P+IA6=z!LqQrUH->&+Q(dQ7D=TkQ zyaCR0=rGA>ku?}Si0_>`wq6z!LRfNz`BE&Ojr2mx9}%jCf`L3SxC}zJt!n4ABPB); zFNdOHR+e>T+FoGT4lT4q2aQN^j@Kse1%Vn{wi=W`V;&JFpQcHmD1F6N*kn;t?~w@6 ziK3={WvULIENWU|GCo_$qGl^hMu<)nH6J4F@nJ+wJGw_9@mDA$dVIH&pKJJ9>bYPx zRwHs&6+4K%L5b->7MK*ccBXhP+lQ=AIcq8iS9)Q8(XDR%UTM~07w6^owI$+NH^D)Z zWOU8ZwFU8Du5G;llT);qT-Q{KLZ{(zP4a1WMr$z0rEQG${6@WPCQ)C@oK20!J{aZO zSk1&K=^h1D^;IR8>^)DxujqBP^R$EA(4X%B$90*Mc}7e`of)B&u-}!ewzPgu^a(T9 zFHCr5QpB?E8QO?-lMWu7E{V3W$=GUZg<01!;eX;KCBt0+QSf&iuq^?(ts!$!Ek)GpY7!i*s z`B?F;Eg+H9ZL06D>KWP6Y{P{)N6nMEuyx*0H+efd%+ty~J zkDf5;MI!A9Ly(%GGf#{8K!G(WDkVAXAr_xxB&ETi%!#3WF;YF@EYeRu0b)2D*+!Is zi6WYROwjYx2(X5ri2&>)ed-`;z>&UJ>s4pDziTI z9O$Pg!Gv;uBJIu`o~HyZ>&@h)OA84^5G9bDyPCQ7Un65#kKt&RL;`VF5E3^Elt*YF zRJjpbR(G=4W?W9|gi!q_=1frCa6IXkDXE6tT&o3n4}k)*Fu2VSPcw7E!=c521!K7~ zsR&+ehc~ij&&eBhXZ8t8jGZHBHfs~ORt;=?y}?wo^-E-KSay?O5hai>f@WXW7R{Mf zlox)1sABi9<&0}ut=XXh1?h922sOi%QYI`}o-p!B)Z;)rq{vK)j2FK|xq*a(HBn?N zH&jiL_u&5iId3g*_~V#R{&Qqz1aAoZSj)otiS`GhGXR&XevW4;^Hb$h*zf45!LCF4 zJrYCd^1Y)6!bV-wY-3Ur@fX zNQbA_o7hCiaau9*MqcZwX$m$YSCG)uegn++jePZyWlcRQ(nZ&P0Q{zlG~9yS zVE;N`H|-L&FgD{GO`7o>8Mhm@YZEvP#10l!7$xZzk3}g#e3AcYa<}Am$ptpfSXpHI z)={DskwE0oIB3?PgB-)%>5TN-M4X*iUYkKU8$#6a;Bj94v!TU;y!6o8@T*a}=HU*! zgnsOf7kNf2^zEB!*-Dyt5zT!@DQc0%vYvJ&m;!|*t!kb%1+8qA(%r%H&@R(!eZ=OY zdO(+X_Mhq>QYEXnL@3mdZMnBm&ai5YU9)m!`HtJv9^74ks2}=@{^yX9GZz99b(t0XKt!Pd0qZE%tGS=FXg zJyql>IfKW>+qvDc97WD^o9v2-tz!$s`T@A8|MmC5MHF%gLz9zM&UdzKLpx$o<^Z%4?nDnZ$*aB`W0?*|@jvzYqoiVM*kibR4%}f0GSZL%v6iJQZsE03z0e+1zVW2s~%$2Z5P&C^m ztHA%OhYFQgUUhLu+Ii09mhw^8R*FGu3P|MRz(YO+Fl2)F{&Bz{C&NHmh{q`1X#kMv zw#y{Y$H#$toC5SX!BCP8?)X@+juQYK4Ip6x(SxBJCk$OGeB)yPH>N{2CZnSzfi<=u zj0qGIU>bYyiz)kH6rk}?;KgLf#p!%XApTc6$ECc4<2L$?MKDR(Pq62-%+4VBx^S zI;g0F+zBA&r~;!{uep<}>}?VkurMslc-Wd}CG0}Vo2ImL3XyI4ja+1MYh-S0FQ|4Q zc)T07o1@}~_qWrEvEv=JpzU~2^u{u8a@7{GpE4AqWNnzg-%A`fCD?i`6jw6uYQ*F7J_@x) z&g_S!vrCa$CtJhj} zZ%fd+8nXbs0gC##nQ7HECD^TiZrE0Vx2)XbZY}_W1mM~|wQbwBZQHhO+qO>M+O}=m zwtL#mx0_A&0ycTBVSY?9bGF@?D5JeTNPc9o^wr~|H#JjG175TgIFBC|l2=D`4nzmz zjq@{1ZKcwxUEG5M>WW5%=^0W*Sl8hh_2!|-Zhr6nc=Am_g*g!|z5>85An%Gr+*M)7 zwVWndH*268c>Ds+uYtD|w>vqm1&{TIMa`$k>FoROJ*Lu>?5U2%cjygluMSf*&DH8f_ za6%rv1nYv_)7c$RADlboSGFEMuxM^6#OWC84l<8)js`36ZvAE$;M#ueb>ihxrgwhv zDX`$}E2kI5zB07M9(~D0y!~+^F$EDqVSTv1rNr>yRNI=v>PYTr3P>4uLioWEd z7>Pi+Kj#b^EtGDn>MH`7cGpP-ET%*{&TvUvLOP$!zCVuwH#!t?j$r31t?WjI<~g-H za-f2U#K|bYB=pgahY4+gE9fF3Q!~$%xR`qla?bUMfKlSe)sHmm%8fD>(?lUz6$Up? zF(lsYw*vM@rp>vuBqoyWD3`?}xD9`L8Ab3l%Rd1Rz<7dlOk1jM6&F<-gpMZDBhWYB zy>kS4T-78TL8y3>Vt?>YK_gxDb=#U-4!kKO=r=X&eE&X(0lfDC4z~EH z0>Jq#`j8Z!3GD3#xK84^Nm@2ZD9Dp(GEdvc%1M@@`6yf-HK^Z_o&|g@y?@XY(S~PM*=b_7$-8EIt?vMivnBK&V{G>@giW3qc#nyhcb=J%)jho7H zTfJy!EAA4y79^$p=aft^Tr}nDyG5uzL3HK2jT)w;A#larW^>^Xl~jGV^;MH}9zO5^ z+Y}bQ4*o0@CZh@xokr(kZOT}y8wfI&Rp})TYangcP(ut^N!L>whJpbj+Ur1tDBc7g zu6YFeHtmUzI=5qR&*@egd^j*hZ)flvj5bs4muglg^Fd95|n#3HSKe^k1!DTf^`*K zKZ31$*;D4WNXK2D^cUTAN`4eg98<05(UC&Cx}+-QIxc?PLc8!;A9E2?emi9c-OB0F zG~D6eM8A`l13Xko-m31ZNP3RQ+f%+D^-N}OU8|b{eI#3nc|p7nAM4)Orcj5ebZ$QP zRoX<^l^CM6Q)LN8yfS~&Aw!jxQsy#mb$ZCB)|C41sbGgq7EVC|a4KEWySzX&3HS9Ukxfuu69093 zG?nj(4266?uhxqF1mqXAUz+#2mdULE1`Sa6WI@xA3l)nBQtaCh{xF9A_@5Imn>7jp z{{O@*%g!0YtlJ&m)HlYH#Th9O;d+s!_o(U^djJ)j{8k>o8O6qq+$q%-;^Y!w;bAkU zv+lne-;>gB-(S{m^SD(5Sa%r1=qX{b<{@D5lhAaew70UK2@2_3891`+o!f-Gxjpz9ovLLE>%N>Y%Ri>8w&&{dai*Z*8+MuDJ8NHDMiHFx zd(1&RzyE4u*yE?b@UqtwELs0c<=9jOUPSgAuLkpH6{nW0Is7aDe>XJADT!=!g)DZ4 zjT9FMk&aawVS#NndKJoMSpopf14b5Wr!B2jAGE?hixy6T8S>>R4X0^g5ik;QJkWn@ z|HLE+Rs1qqOVYJ*^8=Y=X=te4UEiy6?;9K@Btz}_>8{Ul^;4z8GkkasuaYm?(&q*; zb%dUwX^J>4yzD0?Z{9qan-;bWuZ1mK$C?I};|VkCv0HNrSBnj zC6}yU?!X$?fG_CFwqPC7)P+^q2xZRLUm|#!$XD3BX)2k#bXH=Ru2hD?qc)LTYm`O) zQuC{rGaX8OwrU#!Gf#L>t0-Cu|Gg~AlhX3hxi-&% z|J;h?q&s<@ymGY3gmJuaxUm!x;q~3Uhoam*Fw~Z18mmdvU;7ZEQ}j~YRxHVa(aYXp z{aMtpLVfkuh~@jT_K;0Ra+*(PLD-1zwzc!oi(?7f9@@j^+H*C-Ivw&(rU}>xsda0S zZeYo2rMAh9M`!tNa{`)ch(`gfsuVDXkk7v~qd_p$yeux*yt>^hd-2qcz7{W5I2XWV z1Rw3-B~)#Uv1nM=Kv4+E>@-CKtq$!hL$h3|x1cTdb8_7k-Cx;Y|hbn7gc^%M8U&{c6|6{^O#(n;|Z)o ztGhLFP}I7)cTpw$&_B4<`Ofv)kV<{6+Gyfp>1C>c@*&T3eXHz&F*zHbm932IsB!5B zcG1GK?b-LTQ?vPa(#q^j{D>R-EYE%%U)+Z3IT_r2cH;6VT?s+5f4OTj$SOCbs*)b&HoByCaMQS|q7e-?$ zISVqF^!X>=|xk5-}$4fM>nf7e$qsz#$TJg~&+5{ff9jir(} zI29&adrbJkHKW`mZzNYHvblJ&@Qh?IyU;6ogx_FrpK4sxU#$shlhN_0HZTc^hEDcV z>plO6CT7p%(kw&@F}+?{0*WuN;OoZ#_E_ECX1LuUuDOm*=s&46ZnF=h{9Wsd1*L>_ zKhAd9Za=W_+KErY$m3`BvLVRsgg6;L%c`@=XVn_+A66|fuLdEwzq=I zhf~Nv=KDwrk{uDeNx)kMed!-yMT0)$jn{pb=)L`S6Y44?4yA@hiP0Xhr=oY;SpMI= zhitIUC8Z>i3=f#yRuv+zz*|^*A<0)+iyWCvl)jp%&Q>;ye4-4!ejZ#^D0{tK&3^L) zC)RZz^TUiKzL}>AsH!?#FV*?6oA@WAQ|PvQJ1I^mWmAMJ8Uz>u5xQP`OnnE$P=yOS zBcJy7lOWAa815cDMcGNob}Gx=v8fp0WOW#r$Cw?#v8h zlGki#t~FA?9%MS-DCl0fhgjI8VLq|D%KGzU44yqpj{_fr(h*RuKtqlW_N-dRPRMil zEh)Jdd-;KAFTTLMEYxp6^Zp8kjR^5DVsEJihZYSd?E^TyejZ+RP2TIjQENY1HFx;D z)dvwa>N}6IVyF;D!9LwbpZ6oFzuFimOim~%^3ShGKKR`8PB=pcCAHu@an=2qZ{uMH zVuCtzPL#QCF#1!}co+}M{6!XTd3mspn_ub4 zvgqJb#p=KvgF%j@kg6Ogc2(#c0bnOHl~xLlAo<-sZV!(y)g$b`Gvd25He3#_of{tmx68Kn*94k0{p$>N2h#%@jTm;vKt6@}Tc&qx<*2a;QbSO%woU3jl z>(gzld>=y_S2@7Ap?O=s3-pK#1MDT5Zu zcMWTiH1*>+89A4sCf=tK?5LS-D2+`$T?G;}7hb*lD;Hj~T^B`}h$=)LFJ$gkp2h0! z!nMDCkkqua>WVu;JxgKCd3~k5rm{|T==pK%5i^)-#+~cG$Hn;hFZoi)x3|y;XFNus zk9?<#Tsdg?B#nCJ^$4E#cP|ls-79{yrXOyem>tG`pPX<#;+%~@k$hN7a$kn!m}f?? zV}_M=d*AXo9$6#<3d(j1TZd$AA0Zqp(CY(kdgw;qB z%ceyIbs3JYiYKV=r(vmMj@>5_vGxwdNI)}0w%c1%lsP~GL9Y!#H^Bt zbkUwtUT7fOHA9Hn%a z@4_w)ld&CA7hoq;>Kx|Tgm5ZtSkydg4S&IPw0A7=795Ykk25GU+@r%x>=(M4%XkDF zXf{`uMh=%v?bStBnwfp{1AY@%67-jpjcSnShK1P(pb6RSR-xEPUbIy4nCVz;o`hic zZAr!+1&8F@PF4MV_N!vo`ZsK)vC*dmV*i111YL*VPk8bT4#cjd9(_2YfeF2iwoDyQ zhrfYSPhAmGF$Nu`qHy7##;$=DOnKsYav!C6iyx)AD<7df-5;#bMcHlpuKNus|I2+{ zaUA^6XCX`Nu*w<_?q-+8;=s_O)?#HtR?vd0>*DQ6`?R&fHC(-A-#>&l)$zqKon%UP z=zNuqysIy|Bg(kL+9h{Q2q$ zGkzTIHA=LAx446sy%~<9QyXIync8SJRIPKF>1c=`+~;FCyc%s{Jokuo9u6}bo1$>p zT;UU+6(DSSL#>IneN$I~{S$#Nc0fbkECF~7`OFTp{vQoFDrARf0ll{SA@X033#-UD zg@7T+@e7*FDyG^1@1@q?F3;$SbhHB{&DZg>=>|PbC-;<>=tMEzgcMAw zfY(OIB)@g~ z5VZ>jm~DV65#Jl~o7nY<^m93lF33qG1$hwAY+Sr3ZY|RcbmLAF+{7AR{S7Ev3{d{@ z>4CWNo-%^r>VNGv7%alFn;KS^nVb3K1NM_KpPj>|Jw(Oi8$ffdEvaCY`M6i&mBpvMJ%j2&SMuu@r>E1j(b;ieNT-s`X(PK<^luI2 z7#S)MQhT)@SWj-9Bo$*&`^xSgX!z_&G_^%AG(=%t*hCmn)a}rcsRJJ{6f=i#Wu6vswm0lrKZz-qT^XhE5$vaLTejUl^ z2Mc#_itgoBq_h?zitse$z7|AIF(Z}{N(y%13WPMQku0|so4ubBQ)j2*9op_O)6u#y zkB{~57FpY7(&z8$Da=$?QIlOBxkWOg=WuQ?eSN)@m%gG0eX7zZ9SW);3kyS>OM{+c zc9OBTDm&yOUm5}A?m3gCq{r|hc{re;(+m(3p=9d5WgU+qq)XPb0l~FWJD+N9tlXLj zvTyY)q<9sC6**n2{Xf1EOU+zK`syybXw5jC7l5kkfgte+MqFpSCOXSbOI;+?FIPma zE8VZ!!{X}=%EuXed=#q`hxVJjCR3C%ZgzD=!Ln_oHx{qNACI!O<^%>D02^-OJR@vo zJYDi@#pLjnCp`IVUE7P3&gOP3eK zO;8v!9L%vViT_*h{WqG&J*Fv(zp$$Z^fgMFWJxp`RLlF?pNQIJ_Kw6OOmR;ME-6X05mg&+%*& zO)d{u(2jj$*8NzmK(Ucbx!@3U%47Zo(a@<=2h~iXE{DW)w59GKlb%<$BURO&vXo-c zhm3AkXAPP3-){;zoso36AI$k@byfbpoFmPC<@6EPS_uVC?~HMB zi#lK+xI7XRgg(;A z=Vx-i1B z8gJTFo4k!JSG*Fr4kQ=2_-y|g&^PD<1l71f+JBqhur|o>+B`1d>*u*Apa93)TV0;x`<2=%)^#%9{;=Aa6iI=-I`@7D;I_Lz#x@8*nnMKq9qe-_c@cq;jQ609 zI{9R`-Z+^1=(O0|EP3}zjAb+DS0uw~aI#4JsqxXyT}FNmiYaJgTTX(pR%Bx#XuE7R ziCV(U!RKu=Q&j5Fx!!$MdYz>EmmgQY1s+v-_SLuPscjrpzIgK2zOx|Pay zg!@pHe-|D6Fhrq{BS@c>oFfd)Gy{P`{*9xqtLfX0XVb_LtmkfH2a&+(^{1hNSwA=B zE(<>$oRd72|MJUH}R z)<+Zl@y#2MR%)y~21Vg#8`y^M5D_MHQUY#grVmCF<|qv_vRCd{V~yZ@#Ha&~)neJU zo-a$P?+7>3!VDVr>HZ=2l~Z|UMkr9@3F}_>dPoEl7|FKER*_2~ZrW2$$TcI;HyjL> z%xuqx0{)gdjuzc zqhxeA09CS&U{hSsW7TOaPFSLwiF@XLl4O}L!}7I%)ryrNB`%(C5mnHU?fq*LF0mut z6~>(w@ZFWZvegbxfQ1UIaDsgl4(=!Kzk%!sV7zAYmL2DEqyE{ZkF3U9@#h^ilH(=- zn*bQdv_y$G&5BZd;7E0Yq;$$;vEpSFJ?slZ20u? z(|Xs_&XopS_GW8E$9AA#_o<|s)72(Mz_5q@(1BnYM{G;>Rj6MoXJK<_B*LSn3l*|^ zsip~C;X#UogjdCchbRU6VCbaa(Y`P;!`6NMJm~Wt(REXjH(X1XX#^=ZLYjGgk268! zy}oJj&Jeg|gy(=C+hcqxvp{N8>JkWDYHSkn{2Z;RIpZ~7ow+nYF?=Z=+W*5)GV-Kc%@fuF^P#dGy?PT*;Y29sftaP-6=w0%dC`GHFw-WB@;Po z7^qN1k2F0K{T%2D&4g;AthHj`MDRLxOTjMes9+3N6fMh+WS&V`SHn2dFWHebPuV?V zYH=TFDJNUA_i?Z=7TOCphEQsp!NLDjo<^>_hb!w)8b+n0w3BSovS8?OvEV*iJ+-x) zY^Et0cyDMhB>=C=3ga$lPqD4=g2ECSKad_ub)%5=R|O&-lj0YLAr}`h`b`}w^@d0EBo)w zXuKcE^EI`t@q@=vP;J26Sv$$(&X*?9YZ9FCoZj4CsZabSV(?l1~6dWQHB zDZq;0fp0+1I~vG?xwv{LC)Msx#$!gF$O}3*^V910lu!*cl&d`AujyGLafAUu1Mp8; zC;_X(U*T_nsJE7a5OVzc`U#PjMXjxNOgQgHVDh`t)*u){EtO<*P%<5DZ1CL=Q4W`Z z<{F6;Uy!>Uk3PV;B+_HORtqhsXFN@Nv=)kY4ViwGTsK5_K(TFT;0KQ>E(H^lK$DB} z6t1~E=eawgCac|0gG)ITgQ`o$?#tfZf+))EV5$}T?NC9F+Z4rMnNE53_afmt7u@(8 zf+!E*bGvp;<}u&wF-ff*xRyUvm6Co0XEH`JA#sa3P6ev)a`OQ?N?&qr{>HiP25f!P zAiNwNj*jMa+zgKcfjzEhmC}gP&g})Q?`^!6UKLB&;V-W~&S;|XUz?TMmSiEpAz;02 z|CUvt{5Ogx!#v+-mC8)xw@JVLBA+lTx*#uB^$UCowxGxP-` z-f#O9nfim3Y;dZlbP7^}DD{<*CpNU_{2AU2trT3=$X?u5Q48>gs`t6Gqcb1=-7aQ? zJd|J#m+!;R?!gmDGAfY{RzuPsNyhGX5AckP{n~_SZ&_ zCC$?UIL$Z$TQ&Reco;nmDSJ3YitxksX|1M%UMzVcb$MXU?PW+(T+!wV>Ay~yQoUTn za$@hP4=Xq_bvSgpiTVb6wk7&qz0oTrXroxuDP_yG9tXq)RA~8v3gFzqh&XQ>J}<)@ z&m+Qypfz;r^`w#55cF;>kco%n-kbDJt_HCIjW9cmRJ2-n!-ZjB@Gy88z!q5$oHL6Z zv;Vzm^bTx}P7`1G0D}(=^m~wUy$K^I`6enM@p(4P2z-&c(AjECut6xIduN7wz_=`$3@x2)l=&2-oM*>W{ zWh-0lO%abiCb-X9`w%b8?yy%Lw$FoZq&w0M7pVGQ^A_s^vh&~MY<_3N(B5FtiiDC4 zdCeue33#4Z<+1Ga@T>0%B>H`ze;UE~Q#5Q87@|&22K9;IELY$KG2}Mhrq87W)<2T4 z;UBT8p3@v1+sOh)9pJ&;*h&3ouRWfIIT#jty{K{zBQ2f_%fGe{gbYJMvO}=%I;J1S z#w;zJ#E#jPKC(Sldx)ww6@Ge(t`((QpIxZlfsMtd7wm;goKv=^*Xc;y(Z~7W)|4Gc z8KG&zh@FYVKC(aZ%0i>|t{||XT{W3A>6eJ`$XO%_WqM6^h^L4F)l%%y#Z)voj6G=i zaPJ22(o>xH220tDanV)E#*y}Sq@H z0)&ae)=TLnl={s!&03HssQHb(W*Y)XO3QkbRDbR8uOPajqfRI&Fg;A{E*p4ODwVlo zftMQj5G5lBBwF3&n7|JH+%NvqOIDX9w^dBAA+j-g>#2Th`Hjt9=*kA^^?7AwjS6zg z)6Mp3L<7w=M|@|<0Q_~lLC^7@MSIE8W!ANNU&N}_3xKnbcF&EQFFOuDXi&?xb`STA zoEjV`7c~&J$$Bb_BkYFHEwa#*9b658ieZ#Ua$EA@q@ez3GxF4WmUb<{`Z}VlxLXL4 zoFv&PHGz_;ecAUGa`Vvus!_9MDGrRa@vZ27q`Oz?Aayu2Bn6H3jgv)5fhZ-{H5JP< z`9^G19I7^c{|=?fBaLIN3W|p)`Zv_^CW5*yxX9yofTd@Cn3ATB=do&X1Ny} zZWAZax}`N394r8d`OBPU?%qy0T-QY&bejPSm+Pv8a>^f|4Ahg+wPxAuiY`Oh{zsxrSWn``{S^=W7u)*edCv|3--p`!~nN=LU&`ZNnUWqxJYxta) znla&~HNobg^k}betryr<2}zW+Nu#?wNb3;7`gF>+(A;UUIZsrq&#-Q&v@~QKOD5ux z?|D7DN?8u-fc&_t6beV0NR-hysT)}l0wcE1+4Q)#+0}rq4X?lHRG82CZ_n?I9^dw+ z)XQScnt$T&c!4Ls{;^z=ipcrNyd<$~my~4v(CAbJMLvZr1n+J>SjA_y6rnjf)Lhp5 zsiVe_^Cp#|k5nyZ`Z4Gx&|IUw;3BrlXx1|Xo@Z$o$!eQrBA?k_yOVA(|b%-p}3tC-UU1lpz%^~vJi zGaeZeKULmuh|Ml_#bX~&Dp1mIzq&{(#1{cMru|xs{QP5psc~t32I{d~lY*1*Yrap+ z(aS|qwgPUk=G7<^>vkHgv)DSM-oikFg>w&kFrcE1R28nNmU87laZvCF1swwM(4SPK z%x zL54z_tyxL}!29_6V%ywzCcj;O#Q~bK1|999~SVA+rhEK*k zpxHRzQb}uaQXw`32m7>-z8k&Q?zG)_L-OUQa>v2vd4^*xqiV@?*D09rJHY2JF|Ysr z>4Cs<5ex*`&%a|$?XNiJXVe`F08^H}^9v&6P|J(*5%c+*7+r;Shk?|96d z{`3T$!Ben0I6Byq+QPQW5V@1qN4%ioRyD@4s``9MEQ~UvhUT<$CcHH`3HR2RvCr}T z=^e8IvY!0?$G$ZDj0{IcT$8mZ-N)n0O0KVo8kP?oi3ZCNhYxoCdnO8wo9kOW;_&&? zkcXhe+TJUdZ!|PcD@Cjgd3y=@l&Wr@#mTJWP&{cEg2#2AZVo}|A&{9gThX{|V`rg= zKS}e??8M2+%vbdqhKEK*K?=hSh9@QO+(e%3^av z=b-4hueJXejn%RuZE4#Qq85>6epq(FoWf~hiW9SdQ?TYiFd1ajW>?jPY~;+5p-eWD@AelL?SDYbE3p~9~r`u>s!Kpln)**wPBRu$(Ut5*M z2oVgy8esplP+U8r*3@c9v$|%Ny{e}-4U7zjy3;FP0`w74eQEk|8CF-v<{cXQyLJK) zQU;v>D^2sqr%E#_Sg*Z=fm}wUzJ9}Fth_9&U4xp1YScLKy6sR_lBdOPeK89Z-^U;V zP0H)p(aZpofqO$>{z83lN?o$*Ra}<kVPw3>SMj2ewK_TxUbU+YWVYf`wgqn$Mm zpl#Tn!yYuerKBJ_p>bPw(3Hgsf!OI>42B7hX@7!69M=&RxqS7-q0+(iQ4Q31>cgIm zwmEokl7PdP_2QwKe-ZuO=X|>|z4`q`c`Z9KVFU5GV}fK{{0dQ<09bl_Y{ZhXW-(+e zrU1vUHe*-LzCs_K>lV=0(;x1MNjL~Im)W1|^-%a%sI(6Z218d8#LtF5b615@K~S5css!r<;HhQ=%AumxOk^%?c%?#h?TWE#prOol?+ z9;{HJp4I3H<<8B7PJF)pvr*B-vnGfG?&*9iCm zhmQtv+B;Q+dlUGjWeqqX=d=B{B^I!U28v6EA)hX%;hk5U*97XH06Kn;{ zOr{UZooElk=`MYH6x&B(A;@5^K6G&!ESl}@_yB&AV7?80r#JqO5H=c+O^JuGxsrn^ zoJ!${cWXPbEYx5#lfn)snMV=cts{ZR_%hUpPX{(S$Io57!RfX9pZDDseHT2t^n<*% zjAd)m%@`OSW<8~%>CTpaWE86tUc&Nj>VH1p{^WzIEEA(@)E9WP;~IS6W){)otwTd~ zaqkwJ6}(-cy#7M`i1!}C2)m5lpC1_2($dmO+H;6s9G~z7m3m@)VpuL3MrtsPE9Hzz z0XlGXY~9QF&g;G;a6}IFah_Z^Tqm1nrEfBjs#C2AC%bG`05$R&=p^6dgRvdOqPsd~Kkj9h!2@(QW|XX_YhLNAyXQg=*A zjWd#N^4yfn4h<@M-r`rvi<=vg>w62D5@ z93tVqhIsbroI(Ax$L~JtIG0j~^sz%quvAeG?S4Yqe}Ls5>DPzDH95kM6&W_e%q})` zL|)C9$}3L<2v*H=k{{-aAMpjNqJ?RpQ3+$x>S!8ywDSqG)>uU`-jil197d^s0BO3a zW`o)fH#BD6h(?^^LDhm5)P9Tw+vt34s7L;By)n{=vdm2mkHbti>O)>10ZYm%4-08E z{fufbvU*&p*0A*Ped*f!7Y`(;Im_I8i%j&s{QGm=#P!jK;SeGG85K6Arr7}F0|twX ztZs4qyY2t4wkpN?ms@gOy_myiSm7*J7vjL9s3zfZg>n!%RYspaWU*D3Yjz(d{{AZs zl#s&*uuqb9JdhjgZlv)9F+_X7^Ftm)l$#VRz}Tn?#K)g2b7)PeA&Zr?r(QRTz}6+x zFhDFC;U+F_xY^1l`pU?|ND@4Z?ten^mH zRF5>}hkjR%`>;N2B5sD6=clnE9iSA`9Z8A1g?$sVri04Zx>jz^$$3@zIbqY0*F!_9FP1tV-jXQ3Fr3$8Oau?H5^IccHz##x@&kyhCHg#z%`l;J90 zz1b5*;M?nHx@HG9XdEiIS3`VgMaYl_ZUODb?XK_2DJTpGr5@a$jVm4BS$J}?mYs`( zsIbvS&B6LITxSU|Wy%P%$y~I4K+NPHlG9U{HQC9>;QT{GL~;bqi!Fs62x}duhTx85 zo&ea?+*ER10{&BR=ul4RD4s$JdY{IJnG5ZjQWQ;Yn2QG*#ey8e5_JgL6l)C?$C5^j zC_8eL8$Heiobn`DnF~31vJHysfX#lkFwd0`I_Cu!I_t;C{iaz@#hEuIa>a%uTX-?9Ue&67@@Iur z(S(F5M{hy-<|v)+p8K&gI zM_XVL{)sGG%!^6GDER5shf%DBN2?-?I+xa_@E3YwW5rR@vRSwq1L2Y=8!m&x=wvUg1f2kC^k4yak)1{L<(3nQoD4(sx zAg!wNM4xKl1bNbqwx@U>;g6xaoM#EhoK`DM+cDsgdWy=qunJAiIX@s z53-N^mV?Pq`^=uY)q%yvyVL(&#O(WQr7fx$B08vw@Lm0KAD)b~*gk9^@kiqc7`E@FZ&| zg@#s%zas18+Q=p~S12=qC1Rw+5^qT|a)vT=oJdma<+R|f2^Q9_b=>PH2IjS76B2(E z)rapb75#JD<4&#)>3><9G!{qq#;|XLuNdrWQEH$Ea(J$x2T%mGVg}i?QG?Va>KG^pQALNO01mI#%BVn-S38>tc?d)!U zc6a<&|3#9h_$HGiy)zy%GIYT-&Lx|URW+#3{HxcP zk1jhfmolSiEuU~sw{n=C+}3<(j4~z9R zDGX%Yq!k*WS56~9Rt3FBXC1mGaJu1NY?h7rxmU@erKAXVxxR}TZ7mQhA45I@&`4FY znnrQC7%TjBW;2a)T1{ikhLp71D#l8?Oc!;YMh3M>6Ls1~M$y(tmPF7rE`B;`wGuy> zBhx7jr(+_Gai-z8IWvob9>(#oX$3G}gvZwp=CI7Lws??SYP;sRoqCqU08t_>72CI;McNAE_Hs#3dbUfr`Xn*Gt=8N((S?4JLso^lIoyzGiko zTu8{j{{AAVwRX;kBx`zgOkgs+XK&du4#{UpKeOZ#2YIQlTBNW0&a5_W=6VI6CPZQA zEaAw&7spAeYoMWYuKSu!Y|k0^LP;%c!d@o9F9@HS)ti1eT_jnqXqm(_^)Ww4;#;w4 z_Epu%q1U?njH`K@gQ4ZrrNl!FVCSS@`h49;5v|@-64DSC?&z&{0$O^ilQR|@xg+`x z6wb2NGLogR|6MjcbdS2zRILJ#3sQbPKHCibxx79qR8$1`v3;%e{a!vhxV#q_^nNc* z1#|{{U!F`YE%p7J++=ugXat-w{C;RC)e+nq{PO;z>H72`P0C>SeHK9B*<6HepJ&mU zag-8>fEp)tw(or7S({ne>u)Nhf&xng8@d%(9LOvGi7S4*yWFE#_*vYc6F|q1263YO zO13l^&2Z1mxFK-)Yod%F;F^A-fK(k$woOvS5Bc|BDCmk~nNB%yq=3wYjc6+NXn`1V zT%0l}hR5@2K)f=J!<>@!|Da+mC|S<`U&fm*ss907ysT(-6{Qw~n>%u54YKmXGg3v> zG{EehWY&oODbHUY36vU7XA}b>DC=ei%hU>1&u^Q;V-JN><6^)34(&7DgB{EhoqO_w zkfby1jAAFwLxE9hjK$DFmm>J!vF+A+??l^oqTOH|rHq6ow{c9Z7+vGsE|H;x3ljf0 zj7DNjc128q_7(9WF~;nR`>QvfqrQwN+dv?!Enzi5_d3n=j{2qe2*dtGH&r@bfmz#X z$x4jUK2$76>!tUh`XZrypL-RT05x=E=9;iDc`fIHqrzq*Z>go8Y)(@ioL@i&Xr(i{ zi`hbd;lrpz1u`QcL&s5;l5>0hvx#rqUcF5*BSB16JYHT6-`^y;m1l_W>41G=hENJ#|rZU;nuHJ^^H>tk*`ouqb4 zqlJJ@W*G;b?~dL0L@6j{^Hc&V0YMAC^a}Fyx-Si!w?M{?-7iMK9;TCoIUm3UJm8Rc z7-Sei<35*Gt85NtVCf*QLoKCls)v!C5mt~lAk^(Kd`*YtEn#WsT@gtel2}W2GBo|R z{1yZa&k2esG$*aCB=9dwx30=NN*+_VDM@GCxFpM`p;(LsR_J=FH%EmtsaM3SWGQWSc>{eD zO>;|lI3UJyUD1rsgj7xo)GUJVGcX^Avz03YFl7~tGAKK!Z0J6 zq0dYa@(Wv3o%5gbz|q_DYhS~}y!hRw72-b=r={A*cKEOPr^-nTg{m85phbM$HXgGw z7=s=-f7>t&Q~e6B8V#*~?tSW8Hdx&B=^p=2)sr|Oidd6k1lEP#{&cTrQEp6P^C;Rd zTP)Vwb#!ueEM19B=w5m2YpyB}s*W5HB2jyN{6{9002toEz;eTy31CqI?M7B0Nk4%S zipz_=1K<5QsihJ;qO0!g`In`Go~Gk;QjZ3G*3JP=zPJ1hdJ75vO&pRl=RKuH72V=M zp4+ZZbD|W6aFFXyu3Z(Cvemxq0FA$#~t9NuXuH%9975SlWU4g$K*mPsxWyKrfIGftx4z0Byf(mIQ)7L>NjSOS9ILBtz;jIpt}YMybIqb z=B=msuF712^=GzquInOXVS%?UQ#n3lQ;(T*I3~yDkt}(WprSh&dt^?gquiHKu_&f! z%XtOLA2~L!$h?5gIL=zqMU`Q>6AkcWF-t3jeYYq=FbAz9=Ar)b@#4kO>|Is;7c_ghc z#&~^cb9ep+?XkS@Ep$L&)}?kjVB+RRV_&K&Yy}*`0@xQe{r59g*%aohK+V)us(){Q zCp2aoI2hvP=o+PD;BvWQ&#-SX%8JTjgzs%n>)G5V;3wGCz zGX-tl=AORqph}hXphRf64r_Q~UE6?L+we~Ua!~%eUorl&BW&k#i{hN|3d&xOetY|x z4|7|?WaEdgw%9`mBp)Yrubj%CIK##qV=F3;CR#hu;Pye%xxH#z=)c&3^M5871_0ob zvaTmRpF)nOOXb*`d!-gjmw1+_rdZ{g&3(-p_KsMlW1gncLQHtHvF4Z=lY1U7X702h zatx2nXzo4c{WG3l`X=^|ds_EM%pMvwfzpe{=DzoNB8_ca`Va?UTP5eD{ zE;|G?+Q?-I4RZMeX?Ik}ICjCp@*}hcRP+OW^9*%pAlE@Rv@Jg9@QILLddViy9FRjW zCE{c0z_I7>t6Cq+o#17OMkVpetWiPR$r6C~Q~Y+>U;YjtjV5E%xkQ)qLT%=LsB3zW zQC85frigKg6TZ^V4`FNlp=s5pDdeTDX`7M{4lh__gbjRg0})bn+36bfl@a)06zumiZ8?+Yoz(p(#k| zTK#pG4>vM)ut)S|SGq9}!^BPAaHQDH_>*Nj?^;#9d7;bB_LE?l?np@Mxg_8A2L^kf zxG{a*p@$xV!0HYYCqU#bK#!=+q{Rk^dBs(b4UJcj$6um!94X7B#|7`l1Fq>%5f(XM zgS6O)Qe;vUtdIXs?$`mIb^LkrhbqO=IKw^k2AZrc zQS;5b&p(O;tS(Q>$0eCaEuz53(|CLGd_TLa&+htw^9z^4d`~v^ToO?!3(!M zIrBGsnI|NP2EsL{FB;m+lOy*XtO!4fXMgBsxhFr@yI57+6Gqah_P|-&XSvjQBRh+o z-M?fV*yl$%btUJJT^gLO2R~!~{cT{)Rc~dW6TRU|z9k}Y&)($k5U<6gPcw#8Z1Nk< z33Dm!;k%kB!;}s@q$8TRuE}P^eG$k>I}I^r**;+!wp)nFf1;|R)*as-Isy-z7l#9H z3&f5`|C@jMoc*^ZT48%?WxEER?&HG2zC+S(9aO2dUBaP)>W)(t z6DR*gW*STgR zE;5sBD1)qn%>>er5NrXJ0CFkDOXR?v6G5m8X#ZnpCI1Ph4fzn4C~D%R7vk$xDhwUnK#QR$tXNG5FIb4iE7`QeILn5FWTuIX($1)5qt zLKRfWm32Qhga_^eiXfcl}x=X&R&CFmjgiNU3 zehPB=2W6Y*>6P>w>bo}$HCsiXyx%S_TnRiyvn^~kGalxEHfCTc?F%}r;^LVbG?~bH#7NABvMz<;@EQjc=8(}#4#xms_Lmn6)VS}-v5sHAg-t@Y x=Dbv`7<^cltU6<5OC12%+Oj`?j1Aan3i#=#pMLt+{XYNz|NrGhyK4aW1OWb0M@|3$ literal 0 HcmV?d00001 diff --git a/assets/rancher-monitoring/rancher-monitoring-104.1.2-rc.1+up57.0.3.tgz b/assets/rancher-monitoring/rancher-monitoring-104.1.2-rc.1+up57.0.3.tgz new file mode 100644 index 0000000000000000000000000000000000000000..31c3b198ad5b29a264ded2fd79b10418d462ed88 GIT binary patch literal 479998 zcmV(?K-a$?iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMYMcN;ghAbP%zeg%f?du*pwMN+a7o83A0D3+aQ+7BJcPMsdAA=-$75jV9^dm+3ee zqd%ibxR&p}1{{X1bR?wiDe;C*$=K6{H|o61$+?9bVJur->9(u-ZxQW23T&Wqk^HPr{G;*2mAE?IwNdytx? zIru6ar~5m&qTjnumwP+7s+ynU^easQa#9hN6P8gTr#lJCILoPmDhq+7->ZseG)8k=YKH*FWUKB*Uzzi??X11o{7X9}&>l?*WmF4R?)@9rE1vUG?p9ubT zwGKWMZr6dS+SkKbwj_C7tY1D~VY{B1>QQgOJ&AK%_2GTD$_+jCG>H0Sn@(BKSl3I) zb|-CZv#4irLH2IO&)!-$^xxm6(|Ipix`Hjb=pv;4ddwDrNcnO>+1p;|VTCPviK=&c z>&JaxvIYDetk<8{qkL4lMF(q;U0jf@cK;nV|K`14xAoiYeDB8JwC}gu_Kmx~cGI`q z^ZqU0xZ}6j@HP8w`OLMnrd57JKkeE%k(r=XL$w3hLf@0w1X~h;h7i4wCMGicugSwLhSI17$-jE&r`};r|n52`( zMg%A^WeQV91OOx@ro~h*svuYY{!@+fjEhPO9%+5yQO6W7F345WWI-_#^^Ed=U+52d z{tX}O%EIcfN{b6D)RJDvzE-ZRY1NBp|1POiG_90x`RxlW+cNTxRsIXRSacLEiw|_b!Ttn_8{KTqB_2sw!v(XK}|%Rg>|!3${~8P#B4#3f;F7$Wk8wxM3C32y}Beq z=zE(B_}KiOqaIqc6KOxEqn^7m*vfRB#x9kY8GzDE%@jQzKOV!PpZ?`3I;rJS^KGF{ zal<;>0gS>`&BQEqWtaU@Ar3j)(aSn9>RcqEb0d+zNZ6ll%>{WITdQhMe;D=1;g?x>l%RY145 zU;m-Zbb59{tM7@R^Ho$_OL9iUCC$iTmhqaYCxYbqn!|+*DpqzzhUgaR%3)3;PAIp{WV4Ee@30ZZnP9l8e|UMivavMCz;8MewmGQ`Twv$OYm@!q?Q zFuP9~Z2h0z&g(zX#&$M0+F*xp?oZz?x9e=)YEyeBlwY<(@y9)QD+6ii*G1R8J9ORK z-wItP+oJ3K=Y_5y%x{IN5BpHHvMPgUHVyvlZ^EBnW9xUU{^jNw@9lrpH(_E_JZt9r zaJyrg5uuK#n4%F;*{C@wBPolZ$}=rSH{B**q!rlZ^q&i|>J%pzWEE7&@RJxwhJ#W` zo>Nx8eN_b;fPl&QmR=I+)L95jeo~(lU<3uhhEL%x!l=JdClB8UP8in!q)fcyOze zw|V+&7vP2^SljM4J9ov!1!>Iqnrpq-m%l1Q>&&Wy@>+ z26jQraOQf!xzWSo;orK5kxxRn_>PsOocRSZV}RSOjurV>UTJ?d)!!QVM96N z;yIp?*7gWe=#P{wf)zI_;%fix{M^DiudrI4pEv8GuvicU6@74PQ$lkB5p6$B3-05dO6n1S8IX|h?b*G_qDzClU-kR@+<-bF1?mq!?Z>`V$+>rZo z;y6Ab4(vFW%FmtcxLFhVsaNfTt0af|fT$%wr+<06=X}z3;Y(CCU;vDwxGMMxeENo( zvSP$IOY~B|G@>-p|0_7okyPLkwLn+_*f}o>ex*NHryU7E5>1vb^&w3sdw+=~G9E z)U&=KD8mfpRFF&+E0hU>6)|h#R7^-M^~6CXAz=ah%7D9{{<5H#gdtp2;DKPhup;jS zcR%@XpAhpqNKiSt(o<%KWlCl1AmFNM9f8IXIBBEKZ3sEUH}w(oo|z5?>&SH9#oJY+ zZ#evG&hX9Wc;i{#;ymBb9?$2LQMC#ORPp?<=QBcr{8Upx@}pYI3AQiF7RL+5{bw@+ z%30i24-`~uZ4wy%fhfACZ!5jq?PUr9U69oj4Z1kv$<3g*VYCv*Y+a=oPx|^FG&HFT zMDdCje6jiq3^E>|#tMyi-BunQdpk}HAbSj(*6&F~5taccjE*oVIXffDP>6{}1$4C} z%)qx~Q$uLZmIq*fmYUe##Lom|0#JdA%U z~wAl)ZoqF3sMjTv0&BkK1GxRHy%&;Uma&h=Z z`;>DcL78OCvvAY|ImXN4$*1Nkp1G*B+SU1>ve8QxImh%nc?gLT~drN`X79TmYiSo1bGgJ z=(xlSqIaa0@Rz6!|73gEL5S)aW+B3~5{?&;OH!D?4TTqwV{0#+nAposR4iQkwqf&i z+8LP{+F`>3IvyF!(Qp^dS8>>>E}2u78|KgU-0)lgF*F3lTsQF9FLjwCur|=t8i?G0 zrUnhk&X^Dj2W*`7pmz$sfG!)`m~yJ5REF3`=^ zS2O6>?QITL3}Ug@_R!w*uC=ypzq8T9kFBlWQ0v(Cs|ADD%r`WKzAp5>n|wG#XNLAs zrii%2McP4JX&2K0Y36_cu_;2q$MX_dbeY^ucZ^Y#Dis$Eg>@cLiHzOt$w%5m>+KckP z?x$Zo_=Eo%aUB|zHH*78z#f2TivBQZaN4fAE!P}MfSE37PEV$T6PvjQ!_m|93jlz- zSZ{tIjdVZgBlLq&iAW9Q<*V0Er}}I}nFgpdlx-J2m$g)=u5zqQ7J{BMa6$5bS`?b< zgjoruXwVf`%{A;37Z|q4lS+?JAAOK*ZQQ!#2ZPqCUp$gt8==v(R=mm0glGx;n7QB) zc?-B}gv!@|CoLa@sY$K+r3;o=Q)ysEaaa`mDs*LVjGw8P6Navqv>?W}!=+=gl+$&w z4tNW(^vSDdQc+ba;&0G-T@+@MG{X5?OvBbit=%_(&FPEIG%kv`amLG%Gj~pZ{$*!~ zWL7S*Tuu?5Ws|-Aogpe$_LO0KaZ%;~{MaPf?LR!HM+P{gf$EEq$!e*1i9m4H(B)j*#$du2ca01W_);H`%Zt~mt}%K9BY!?YmzDJeJx2Qq#1=bEbQ zmz}yVNI4@y9vuZu5>NgTE z$Qj8v%jIjX)>eH1IdPxhY)LlOIF)i^`Pmh&)>Lq6la1~F%Ebj`i!-9WSt-)$>xAd8 zu2{5E5W!GU6QOx{iD`jnw192wuk%eUmePEvL=aPbYQv^c@!Wbsgji>ihW0i@Cng<{*s3y;G9cTAZ<7 zdsev7qd_v`m2QZu>ZLaen0si72I&_A3*w9@;1SQ~R8x|3O<44Qn)(7HEzjx-&!GGn z7mk~4?tnQcMFR4)gDg|xsOJzJHd*I;>xWtCK{PV08ohhb^wl7urf|(LW1VSm?R0?Q z)@ZhaczBC3%8F!kPP1OnXaTy{P6wcpwd3v~&#;z`INKi`(rmwb!fb!vOqP2pMnrjS zIgsSFrG_B)6gT9!DQpO=`?j9E_SI;JYhP(VTKkF(VGT-p$ZAjuL^b$rNoxG1O;CGF zTjVsT6B5(T=YW)U$~+MypTZf6vE6tXLqR_52ztL8>h5Rpt(P~JJS zzWpfEo}`RvPA+`~#~&S039{*0{vx*(yp##)9Zrsq_!VnR4hS&s@c8J6fW;z|b&p6T zNQM>3(G^~WNuaG1EZpuO;OfjiS}D@#85$EAQ#=$UQ!PslgAIzOVYX-3ywp>!5)C(* z(}L*l$Wqa%`4Lt;hZ6VQi=_T{V#f`2%|Dw$gvi@UfLRKC5mX5E9Uo}+Yv=yLlFOkP zyLmCDXVUAY98Y-Tbk|OHXHoDOE~e(S7gacoysSFD3f?+^kWqly9I&TkL8THa@3=M` zqQ4qRBYsZ^O5VZA&!=c z=|TGFf$!}#_eRJNveZ)v#~^o+BxKXd>*g+-twvs-A-`E{nb^KWq+@q-Wk1IF#BXat zx%AbNslvAIGj`e6|f}@uWgstkLusLtg@%$xMCxS?03vAqB8ZFrAH-}FGGn>8+ zV#ga3AXiHwWH`%60|sAol~W$L)^t^Ex(a$?r~y>9*95;Z9hgX8MFh@NCvpChQ@Lvu z{V#AOcVS#-BojnAe~Fe_&JM!ef}6$xw8}eVJ(5wW1%(})*W4X;m~OBJm8{w_tsll2 z@W(6g*b8{)row4R3}P3>oM&>Bah8!v$x#K{JF0k|fb$botHgdA;;O=;^nNZM3alhq z#>?tMLtROa$quoMZ4-J+U4s-C)e^V-N&#<3ej>QO9*yA~-TUI+?V#8_51>rZy_{zk zM7XjBqFdYsu`^LC8AYrc#NB!s=Q4^~wV*>^;skyY2#qzT_@S)AWI}PQ$n6<}%YyDfe2jIT3^z%Qt7>;8Aix z^a=~2+~~k|3%;nD25Gjy^tIEy>Ut1HVR->)1BN`3XoSg}%rw#v69AV~f?i@p61QBV zXci}RQCOm2Eu2HoG7KyL4a*FD0OfF6bhW+;JC>fuz9PYW3gN%4s~2dKy>Jr0Ma|nJ zP;Qu}`#mg5A5C=3=TCv9h41B)Q)U85o8oL9D$?}>X3HAGaGker`7BI)lzXC z_Ox50XRoM{h?^lB7bffCW{;VcGp%Q106Ik{TuQ3<)Ot|Cdk~Xdbh+odjh~owXll|E zV8gm~#%zyK`XMNC^WC}_CAQa5&}m#>z_VwS;fA;-v-31!2)< zIb0&ZVdYec+BnqD>KyVk^zPu6s~~EY1aH%XtCG(~LqpFvdBuf>+kjH(tsvh^}US8LyArbkjrNgA~!hKdF>8k)wwP z`;Q+5wCBhV{u>zl8c7AAk}|8BEQkvdDe>Dk3^!lBX)D6Rco9sl4SLzjDr`FQ5tz|1 z`e~FV01y83!@D9g#d#6H6uj1dC;G}uNUkrEQH%5$MYC5GAnWE|T#|FmWu&4dX|3bJPC^S*M3<%I3UjPkySK zVcS;T)_E`f4vV4-@7;dnw)fi)9DTy^`UdE0bj;@b*lZUUpJv#}4bX2Go<>z4X$8u# z^Vw>O{uXrf_o(y=Q#D2VV;OyId^QhCkIogrDkz!kEqzf&=p_+TbdS}AM@$$?fZ=PZ zSvTj-RKuAL{;I3wd|@sH2g$odn=2y#>mNdLHSR%P{D>nkM|bo^s>-UkcN?07?qi9x z5XBad4SyF^LJLHilhOj2QE@_Jw8V0WN?aMq%C<5WllNe*jcTn}?P>{6RfR35GH{9i zY)N1^rGUDhU}+?w?Uqz61+N!N^c^Y6d(v&7QH&0Uh_D4^Bz3;x8(>;~M7G!*hY-64 zo?4RCy`nIApX_T|f>%ToD@0_5E0QD0ZC0lxSpmPrxH8J9O2AD!TNz+_%src=#idj( z&GzKvk`%n6Y;kS{rXjrhn5ZR5R8=JNnt^BQjk~jPw>2)Gn;p3&AjXvzNe!|YK4PIbGEbm2Hk{!QeY0}X$*ynk4C?Eh0Z-oCzkktp&P_zZyxD%I zaE9{_eIr@V=k%>VmtaZGQ*=h$aZWuRAw1)k_9$uY9oVg1g!3G6wIssAA`);G=f*%1 zmF((1q-IYw6AqrZ23~((MirVBH5lKxBuy8$q!1rxtA|6qxzXO(_#C6AK6j!YEV!2z z^`D=APVA#3XdnxmU8J^6s>oY4K*=w^1a%vP8HCl&%eqjsD#!q(C~A-(!g2u8+v6G_ zVW(gG?c38=Z%$Bxj)=mvkbiI9Re2Stzi@#?DNSq)wH^)z&y=8x1<^MIM&S~s@tEiw zEfl(65QfsgRVn=*q`n7e9M0?us9{~ire6ftrv@d!?K^^U#)TsJ38q4Za1i=G|NH+# zU;O+tN?+;=_m^L$U!WiKPv{o?^2-k`*m^yCmbmqa&QbdGts;Wq;?3#vXz0%=yU-X7 zl)yO^e4%%DU6Y`bot}9f8mE%2FhvvIwRrN*sNpkZnv#V z17w2{6|%2=yH@{xSn!@p4dI+Q_h`HO-MT;Aq6_!(aoB^kp-jI(;aq!aZU7&sU{YC^ zB^Iq&m~R%DZm7LwKP8b8FXB~yVZQbB(^GrPeu7u8qrQ%U*RZ?&t!cGopVp(|ycR{& z=V{{uGW`Pmx!q~p?0PCIDsXYmxFY8j&Zqx&(>OrMWl^BN?39%H-n(!3nX`gs1P%Pb z7=#f91B*k02R+mE6tzu11K+jHMD!CA&h0*#MG7^N+pnTM3IjB-$T~oS0F4G{5bWm!#%Ps+QNouX)ypNt)}4wJxvTJ` zz*4?!BI3>h?}S(>IV<>Lv^U-xkH!y26`W@)!+=oj@nE|v5qL+hdjNf~T9X+1fM!I66kxWD+|PaE4B z8Xt0AV#?s@c-PqpR!iw@aC0vFavOV50!HBqtED>vpycT+&sjyX6ck0q`7j)oZXL&@ zj(qpGJ8~KlynT2-CyI!YG9uBEU-^ZBu7a3jqmjxVxb8mg*_#oih)DDx66jftn;X0?po)ph^*D9Wihr2+mg~JArhnNm#GrxyhXot3A3y_wlBfd29rw6X+mH zT`;-umBA;N08NXLz1?T)_dJ){bG7Th@jGMt%w6eM@ly!Y7I zT}sgt%sl!jz968m1vyI3CwLSqy!%YoZ3?<39eXcP*fh2lNpM7=r{A>NND_Q=`;W?< z#Bgf{+upLx#nVn#a|3BakhvA4_2ZNH;A(ci)0J~!Q5+Qz+>gLW(i~!w9cTP_-!ngf z^P$NRW-`l=w@P3ibp;Khoec8ogy2qWqA>Ed;*w~21onvWoOr2&6)V#rox8DmWkH#G7Xi;`&l@6 zH+kjOQnO_>&B@4uk)t888rXzZo?beDfq1VM&;$ilWFwCFNe8hWx@|;yjS-jQT2m$a z=8<6utCp14+N+=-h7zXLi{R|7gR^I*>E?fY(hRZSIR6G0TF`TN+x~n8o&~3vEr^e+ zvQB1<>q~^rjg;w7Iwz6}WAXwa$ajpCz;i8IDsGa;8RuQiFq)H-Txl8CCtwWM*ui*h zKse8Tt9tcgve{v-u#sm=eBq2y`X0_WV<{B_3ihNWySdZ{mi z6P}wBqG;*y@Gt}CZ*=bov^_q#r)P8TC1)f$7SiMt_L+^5uCaE|Ja-qc1KS3GtpAOt z4NL1n=Wm#Q8>4^ySM=wvAAR|7Jif27c!F}gdZ2g9d6+q1Uw@4TuBX9$$EIzZ9(GSmBx37`+HLN;MTnjoqy>2ea>+{vxLL?Bx#}#YEgO1o(InZD)bwt zOCJ#VTdw3D1z*Jex@!{R9nZo0X15E-2`P>xb#h7w+;agjl_=(}n;=H$(`4Tp^j+aL zv03jG{k$vAo`EvvIW4JE(n@`R6FZaw3OGZ|@SJ>Z2bue1bdi*Y;ZkS~SvGx4Q9~3> zVwJdS-1G=CL8sT|LAINERSo#C*OQHtv&l$$p&Oyc&L%Y{`4SftB$!|~eL&(~y7yH& z1~(N(u3904(9>3#_^rKHUWcb8$MH;->MYCG1&HA}b$k+SqsZA&vO!+sGOjXOuH?Fi@ zRkzf?_L~0ov$u9xlBa#lW5bpB;%LG~LRcgANBQ)d_&v=f>a#P~g05y5TZYLVugVX< zwPB~yb-IH(gK6?KE6j2%a>Qppl1$x``e5N4E1aSi=ETo=?}&pqpC+c$4$mYs;7kcR zs}&d4VRA8WT*okhLUW(}dKWU{#H5gHCe~ZUz|Sc)czDBWn~yK_bPT z5CMbNyH?{Vm)HykBL0FLGM;jTZBI1XfR4xYi$$wx*FoIItdClvS`#rI>@Uo?Z-aTI z5U*j%K0Z+|`{Bv4mJ{7eHBlei1u_P5?h9PCmN)$Vp;x$PX+teq(DfFJY_PP%RcoCy z4<%Y^1N_Vz;GN0GFyet-&Q*EG$@^%Lja&)){OVl_J zY_&Sk!niO1d)EBQ{FTgJKR>e(LQW8NMmK$Xmjn{_Xz@Vv@@JojM+xV1MP~LcL+Fak zB7z+*Wx#38+#?a@LMVw?5Er;^gLtD&(9|+MOdYsq;X4qv_or42u&56;WRr%~l#q2Ug-pTpcdj-;zD6L*6ua&&Kq>x@RIc>|1-mn1aIa zNta_56@+i-HMzeM6p!4O$-fxYg#TC3D1IYWJcxNSVyJlhL1TzZ_e%hX-8-A0^ zw~d!=9kx4!RA&5=h?UXW(PGy;jg&1(c7ga6YbeqU82hsG4$M}l+dN9R7Ex_C+u*B* z7Ne~Hya{W&6Sy_{MkD<&Slsei+f*aj+qunp-?-2og^ju7-B+%Q#NF ziFx$x$SYK6M!%QWT!W>yQZH`mN$ ztjNL|P-d&Xvun5kX)K^BvkbUQ5m`((ufhuxwR~nfS-TpwBAPs)fanzSJQz z(%jE^_f9VO;<;`+1);PN6zD&QW9rD;$`DtnKlXW0yIhgLQ3bAqz;+WYh#ChWuwcW! zH;_EUmyT5|ld*G4^y=ns+%gD~Uk#opa&YL$D1_ z_QuP5CPN(H1_e*&#ugmM@19ZQnMM0&p8YB02Nm;GH z-K-Y!LF;uNWd!vp(gSFAz$C?Wn*eMi=vVZ*E%_CawnF5OXR`$LGQ;wt>lYhO6*;oQ zOUpGKWS(wI@4XAr7c zZjO_QcVI+M37af)#z9-@gC>Ng;wA%dw=SKxI+WI4DK0gqBzTDg=SChzM=#HSlBX_f zLf|#fvJSry#Wb3yF+f|-ww9y9f=ZBez>u=UEHA9&(~Q_ebL=*H|1i2#pGIfap%Mvl zB`8Sk0MI0`Ff*|Gw(IpKIMZY&f<`5h%DkahlU+B5S*?8hiZu%p>G8m9MiikEPB^w* zXbKFr4t=lKq@QN=@u6?xDW-u#>w2rxM=~^RZMLz_MuE5RZ^Lk7R6>@;1AQC_Io55; z8l_=_i-cy79tHkEmf)>(!&`wD{dhapaS#livUqDX#ejj56N{2|bX3T(pm-2e!r zjx|oV7+c26s)lrswxdyeB}Rl9*^V3EER&Z`Q{wRp>c3`H4GE&hO+uWIl7|f^y$yrd zDMxaqQ(L1tG|#{=oSz)4CX~8;7^nD4Jp1G675|y$F!&gNo*irPm0=q za7H#v;&n5mDT-UVc$P-yl#(u(OD@}5JG(!)k5}3DsQX01aqb%o{X$Mcz4Otb$9^+B zYpCn^J@OovL$;$Df~5_nQMNcX1{L$!JhdFjsWI7mO+?vJaBNb4cm+s2S+YkG{EsBE ztC~M+H6a^xApmT$=KEfBp_xbzeFZ^i3(xP#@cHSjY4Vzl(o&L2Fw2t$XtZ15Q`e!9 z7uINSK~@7ljqtH*Ez>n6unm0bllpNvx=x zQhwKY>@=> zo{3X^7WB*jkM>-~9eYc5%?>5x_L;&!!_O}{JLOzO>5ZY_jU?uo3s-&K1HvXrx7o@A zpLO7{Pfgl#tJ-MPMBug!@79ANrpNGR1t9O>jS9}I8O|_4r7yW6K4u^E?U5U<>-Ej?E`ZYo3oa$Cf=s*Z$B>I=7Hx*wyOc}wJb^@%{pcJ= z8s`N3hU9ete;L_#l4_nnx_621R{mQ@{7@l5JFz5+_x;?u&H>)iz#~jE{Qsi0BSwI?G7IT#)>Fs!V%XWg%eD5>F`c2Q0RQi6~NdK%8M?v~4W=qYR- zB5DMG02U7=n$U@$9Ih7#2}g#XZ2Xh7I2y6YwtAp1?DerfA^pn(iUU3m*C{;pa7|Uj z-A*i#QO4K~%Vli5(vKWbZrYl9Xw0{O|@^7ns3pdEIi?|44y?qqM zjHoL@n7eGSQlzW`J~5oBXEqU*=v#hdIpkf0AwsONHkg<^Bjr_CSY`aT{Qvy#|5rlQ z^hAGk0Pa74CqGj3@Sv2tXn!ns(WKl(ljYvHoE*pp;nd?fMM!LGXcT&FBG$qw+aNX* zbz==H8esz;_|I>)!CZQ$TFYfs7sXavT-nOMflY4DE1LT=E;3@CjIam#1&1guSqsu~ ze)9ZTPH?8+-Iee~H&p7Dpw)FYHyfU3m3mfR<5%j9eWV67A$m5l!lqrD3vxZ+OmHbf zA23@Y`@7O~hTzOvCg;6#Z6Eg+l+m&-!-yhlY}3eYw5k{}aup;dsCXWPg_z7RdO-S! zSRV_jWFVzv0?VbIDdeb7L6#|_T#__52}2#Fj~&j&=)UeU&_;C{XpEDbGyOMJ#>9d1 zm9-I;l%49IJ_>c*=Z+n#?akD!u?M_f8oQuLpsc7%Sev5zE;`sIcUD#Lpmja<=+r+< z=K+E_1CY<>h4V|yGCeS#W*SjgzHW9Z2j`Vq5^+T(*){LZgvoX~joS0uekIPpKo;cD ze)?7kY`q+_C_4oh4GFr$g7R7#yEA3_+&~a)m)=}`~w*lp#ES}mvOddIQQbIzxKz?GQ%;nlNm z_y2P>tNxH(JRd*%?&$D>|L1=l7vDWjvz2^3_AN9U$r&MqB(~+zi~sqbd;jy{Pk;FG zZ1x|=^Hu%)Kl1c#$ww#SaxY8a`I$FhVv6+UHUPfIw6riU;zaKvH8v&t!BsM_(iJWX zcvsSVo_5}uL5+Kn%Vif}gMEbOp499oBr|P0?&#a+)$6BIH+ae*jZ{D;G^b+_h%#z( zZMR|_rX$c>q;{0RL4lgKDB{;nyz&m%DOa5`YVZW!@N+5zjTq;Kt3Kyqac;cuzNh5M zn%H5}ZDp&SKQH)#vU5GiDAc@G%M>%tRwb{y&kO~6kF_18fbJVowKVS>`R5DK3zu+! z`cOre`#X9-$+y)v!WDDk7UVf{?WD`P4n~q&OnwjO5{~W0d)aGwpQ<85A!mxk5*N5}f(cr*Er*3m+8$V$C^# zHO~_}uk|KGD|TeES|pn$&0;k$Am$~eBdoBq54yJ^{U_wT$ieXk(vZ4WseFH$oS~WK z&)n0$4n60+-NY%{qM^9?`+>L*A8;HV$)aJ{XQ?{)1{Zz(seL>*iRPif2l1x{BcfvzdGcd;XTclMn=N8wyl%*$rFfpWN+;*zYJf$MCx3Lm&_n8||7 z6`GMHWpEblfNEw*ax+QI;h2pvPYuK;9bo%jcYPSk%Fo)pAqAZ*dpiL`nDNHw&OCUs zM2j8-v$}f4-Z&UBj%zXOADj`dxLt$ohV}9&j0KsZnc!Dae-vRRR%IdXBASvEJ*_oU zIC@^cCFP9QVzKt{xS>o@tun`W<bS=HI3MSM7+eCeP%-fhowiU0kF<)ZO+j*_tBlT7P zlesMesqHWYeVyyAzp5%^f-~Sj^@=vKuZ>YOqB_olq=h}h>PhhUoi%?qi}zxi47{zS z>*PRnoK%o{7e%Rhy)qB0C9I~qfWmZomVvk-E9|0oA3j<_SliupxnSKow9~*M$N&T` zzn-5WLW2E9q5Fo7KMMlEO(XcfuV@n14SP7p#@*T^b{+P*hYs4G22OejyhL{zAArbP z)Ip{HAXBtwygv1ZIc2z@|FC?skB^A3xw_1fN^oVouXeV5Sy40KpvZ41l9qWhYz?40Y6wPOEdWGcqiMb*SFiQoC%Wit#T0%UQPGV=tcN#j@-;eU zxtFTJi!)Y*)mU5O#{}1iDDYWj`Hj^qEehZRj@L1R`p^3(c)*<%K8u=7(d0mG zg_&LP=|J z?|HB(fo1Yf4G3UlYe^(p5}X(MW_39*f9N>+lKcm+Icz`#EO*e|E60radLJa3VLT_~ z@jjXC<=I!+9>J5n$)ow?F(!KlB>Qsq=<)u;FUJRu$D`_E5kMIQnM~3JA3onZI7puF zJ^IoP&&;72zD)}!dxYCHq`S3i>qzntJ%fDRpkD_w2;>fVVBMF!_ywv+ySa_V zBMorCS~)aXl9V%&LuT+s8LFpn76jiSMBaj~R=gHCBggp>(Oq?jDV!TWtv8yw{0~f; zEDmsr4TR?|U)9unIj6Z-YJ~4ZcL-}h-z)*^Nf&`^Jc6COv8?9ik*YQcr%|HiHa&D1 z4b9074Dq3lN2cK>{(M``0c_(Zh6yAp^SVeWi;=D3j&mDS6+O@#7{M1Ex+T}C&u>y$g1ki`;*N4Q^5{m{Vl)ik@1TCv>r5=X zT40J1!#73Fn?!#D{o*0$u>-Ese+|SyEM>dYcxvT14U z7qt4G2s&Ryef!l98QjP}LVxyGsit^8|BNa@nVO@)|0RQqG?kax0Hvqqd3^sykG@_l zm`nE-x!dd$pDEVuu&ifB(}7v;Bg`xzw=edi87cTxZ<^`GF#!ZW@GkOFk>elTCAeJ@ z=|^KPDZPIR=||%R+$TsOeK2kWx2-d18XvD6bgMi$1)T{jR!JytLw1}=y7w?0C%C96 zb4nh=6(Y>gnN6497(gj&t>=3=&3Uw{cpi{V@9A?5Lv<(FNuiDZSLoaZ^@8aoC|US_ znmlBP&*y(?iZlPu5i~UaPc=1lys&Y(8${0%%9T{4bWdZRn~*lKGEWow=dOT(e7i~k zMA~eQK5}a+DZD_x{4#L-f8!w6@60y#ml5kbQmEI9t>ql%!H3sQxQeK?_VEgbtuC<` zWjI@s-u8BIyd+-c?DdgtdA&~J&(`x*e>bT5z4agq{V>dd419}a2eoXo_$}(r5J_37 z)e&tnI|?Z(2EnyJ3zFKr=DSEj7DEMYnMnUzRSQBZr z^thEZH&i5T%&WU0t?rZ4BYpD2thzXq|9Hrp4{sOiXWGF3N!w7Ofq@#WCkLm^S}0TW zf-aVz2x}rWEukhN6`U)036W|^M0tg1Xb}xGas!u)&gmf|TZz5;AWrsv=(Jv&)Kj4y zrm+@)gL_r*72L589AKMQ)v2%JCGS^ieuQMi=7tGd8zTV`-{ru&*1rTi#`B z^y<-HBU@uDd@>H6bP$#&FaU(93_F{y}D&jc>X6&Dx%Pg7~}w7?*N zx|lv(6o9A~fs+_7E6(5vK)@*>cZS2Arb2u&WY;lc2LzS~h)XrAp&(pQOAZ|1!-}54 zhdwWl2GPl zEL8{t>w1Ff47X^TOf?O%B*^n~PV*m96*ryVdxB}Fa70IfSCCIZN_t6(6=IyqRkK`Z zR%aK)IdG#OA9XHxBLdmG)i*o0*V(!A;Xbz(gL4D#Pxr^^E6NhB-Q_Yb#T3Ry??!*9$Y`XewZFye|i1Y^`q;@*OPv?A8={M^RfSfuK3kAb1uLQdsnHhuVT|2>``u|3T> zpQ0HSH?d*`F)Bs-?_|lPzACcA{E|xU*|WlUGp}{OweK$*70u3iX6sNXoGpDCZp(){ z|8&c3q|OghM%U6_`tZQmzg$}L*cps?sW%Sqf-ZeRYwQ6F zMKblk-RRyoX0`&IsA$KpEji=DVL+4#xvvj%2T3Qa+lV<@5~*-m_4gL|1a1$>bCKP_ zzIK(`1HNOA@7Uuz_V|6@%&~TaURtjytziUA$}pS61zyMvUbX>cnIzug@*b=Mm9i#s z^my;fhzm9+>?E;X1a@OThNu@GZ(h95wNMaEm)`e*$miQ zaGckB_{92*WLRGajSa{VS}aKLo_fWtHPGML5&z{pi9$E)XKqkd-!n-kk9NC*A6-sx zQ7!QV3DALoL9$aT>gOcZ*>*tLV-hU}JNxA1jn|%gv1+Y2;N+7{ za@Rc%{5-4b^ZVIco~xzcN);scqL<h2V7+v(1b*nQM9mK=WN)zGeUEAOCc(U(#--vTJPY+ZjlLZTTBc z{&D{IKR)iW#UeiR|0jx=&%rS&-U8C#TpqgY}H%iRY5Za3J8d3Rg9CWz)!wlT zcPzt)Wf{aBnYr~-!5_zb7(O!Of_kjU@QE*Z(S!$eBkqST8G~2{&}FgmE}M{NqT>0k z&1SGd$0v}q%Q_;@c@4=QoO3N{J%g;baxzXQ4=3rvbZ@WoWxw;~VKY+mutx6$)5b6f zNp?+AY#X!Dggb!#*1j0q4b|v|PMLmLT|o3>&Ad6%7&E|`)h6D{QNCihq!~n?&#*N4 zDNNsC8aQjjKzW#!$MweNUI`7urz~tbN%!_TSudwBz8%zx>+m0kE%%&Z?27}sm}cjb zAlr4HA-Q*r?)>=f43&3=%HNcs(ol|<_N`)1E+PGw*N&RWJ4Bufo8BWLK9FrUc(gwB z5_JDEXR7m9=(_Ct+fk=pE8v>?q3yL{->jpH$;oKqO3{5ow~BdU(kh!L$a7K<^9p<% z*EP~VxuE|rQC{$rYSY8Vx;kR74r9#)a&MRL?1EM%i3$;l&h=P{K~tBYlCYR3?epCZ zA%=KjT@k~Vu^eo+Xm@LOzHu54p#F1-Vs}oA|E#enA6Fd6xFAPNrjVkTP@BVzN=ivIsLMs3X}2p;Gz8z@*vH3 zd9liw?6x{s)9T>8S(iIT<&IIgV^n@qjLID=amPyBu@ZMa6ydg5eq*!u*>^8v=jdR_F9?DK}n-`ltEriL88|9FD{ zfIxr0{Tmj(o#)e7Lu|fGmuVkX+&%9$TcLvE+&z&AIp`@xrt6_hZQWJD;pPP&thfHEAmvR6s>aZ|1y(<(686g$5O`ttT|qIxmOvk)Aib3Z-ZvL8GN`i-c;d#=6TAv=F3x!y^xzm5_dNKRfCjVJDz z`>>q{u5bStarO}}@8|PR<8ZtGQ5y<21;3oZB7e_HLZp=dhX|S+}Z( z<;aZJuj-HH)U16Ar~Qz8H~XM};?aSPi>G0_^;eff2%0-1keK1D_Z@R#-kh{F&3+7? zcs3^_WRiN{{Q5YY5^A1Q?s7{XpI-{P@8g+1D%TYB-N!k79=ua9$R7UbGvc7y16{{O zeNKE-d(3NjsqpOCx7}`Y^!y9$YffJ4(YWaNy(bSB^wQ*8pd2f_ z3rvxqxp7z$mwMl5PC}|4^swd;gpbA0{5scfFyHlT&6=zDLtn<*T*vRj-u&t-+{fg+ zyMk(sifWyb>LxYSdPUV*Rh5<+`t>2r+ki)a2GEs%{lln?JgV4thM^8{K>8m~ZQZ`e zZnGJbEzThAw|@+g)o=S*4*ey^;)=4hEXi<)o^g>8Gu|UCmottMT0Ht*?3hKtr^?gCWQ-6ei0Ll$pcrA>@p#q&ty1a=p0 zfO`>)dxlEty>j%qc`n?&A?uT^NrhQ%R;w&ILy)#kUuRA7wdSFDw*nzs-H6T6DAqBc z@Fwt~e?U4xNs{rxh}db9AU=gxjuWGPOvu};sB`l5{~F!T$>nt=cy=vU^7;~sQ9);; zoM#tAjM6{)FZR`iQToRR_y2G`da${(xlGimB0)#@ajp1VUe73-m+HKvGP|Bt^J|m8 zd+-0gHpn%X*EyNh=I_f&!C$JZ=JI-8OOmPLdRZ@sDrR$eO(lQ)<(K2@@~4{ALU(UwU|(J|Z*!sT1&GgFe)#8P z>~DCQVhb)4I8(S=HD52!*<7CA*X)tHw!9i=aF;Nt?~MoN=^t;!b2j;D@A&`EhX2>6 z1=7$(Zz^hXb660eT#7pHfg)oT^_5-G9`z<{)Ei0Ay`O(Zij-BMS@40gZn!;>3TGGT zye1lM=$-|#)}-LM>8ymdAv*zg(>sFNUxW~Q#0>b zG#T%rv7y%sEM|B?96#8B2yg_sq?$y3M>R{fvSfEMw&OWd0xPmuIa5?4QFv!lTLW~> zrctv4iCb87IZpQ<+Z#ZypcWP?Fl@bkeufxU@EC9EpeiQ=^jNhKTob6QrLr8p^F9T+ z!suH3sp17@iQRgGFx(f;ogE*L@-tFHL zVWk(> z6<35Q3J!4=yi3X4Jbq;KX$lKv2&!MCzuG1866E*5mcaRtQ}hheViz4R7#GC-{fZrH zBq$2a#<*rZMJUmH#in)FKD(3T^Yk8ZCR5u^KO?p+s5;INl=O3(y}g_bH6_}7ku0bPV$;#G(?B!MQWFn4MgPTX zI8C)X?&cg^(wrC{Cw|3T*TyuCmjvfPhG|K3$17f=D=L@qYru?g!vsXeKZM1CXL+js zss#TrrF^7UVTw)!Ia{&priw{U3Zh6-S(N>b^NX`O%LvJdjdAiyowLxLz^J?Yt!#LND+&COxadDw# z(}d3xwIs=mzg^2I=mo=g_@ZIX_5<8Gl=R}(FLsQ&rKCTr9LGCQ7@S0x`7Dkn#u4jPx!(5Xq5~)55{4a?3-}o5Tj>j1l@2j&pFO#xWFtU0v(_9 zEpy*04|Kb4Y`Z~U$3;-VZgpSLnR!+Uvjyo6?uan1h%`(FmHs^EM1tp>ruw{A1d+Ff z3?L=&d`>f60+tHMDghY+4bml7n!oTq&wT@TuwENbjzxW907T<+slS3 z?4oeR`uf#CrcHx*m#9?w+B-hc8!CvDT3S(-Pi^VJl=k&hik`rC^h?!Fgg*G3#wL63M?f!(kgg=R+_WiEN#`U*t#r^ zz}kVH^96baLalH4MiNx;#k^GLzGVU)=!5p79`;T&oBkEe!Md66n@%$xMCxS?g^1r2B4bC6EB#cOD-k5CfVPQ*8bm2`*vsF60 zUJ4?YyvV0$KVT%OqPRFB1zvg8!G5#K6%mx@ZKVRNe+~JY0;r8J)6qplYIB^^^)nN$ z_~T6N;G@pWDAbH)O`?Bw2C5!O)~enD261ScHSg*plvVW^|-`9*}Sa)F`l1ZupGva%jtpH{QHxoE+)XYSKLJyjl-_!iSljEaPBQLd; zo)T0^y0v8<+pE9iQoS^+WmgShX@uc0G(#f|pP?@iESe+|Ia+U7+sst}gJvQxC}$m+ zNEde%DMqg6g)?LMjk2$-=LBi#D?qjF`Rzy{HUcmSW_hA~cb!2tMQt0>1zAne;DW3M z0oJ)2Vu}V&f2wg2mVts|iU#m^SU~1;lBp>gyyPx1#-MxYTa?p0E7O$LtJEQWrYoVm z_xfPy7erpY?46}?BWY=VblpQ&6nsbik;tQ3P`0pBrEFmo$@cTpw>h}9qw97o8b!2Xh9VXaS(wsLHJwm{^9fI!4e+NO@GFYp~;!PM2rTP zR|@eOWMi)@qilf=PmWP0NN((CvZG?%tXOND;lOIL-ix(Ih}a-1P`1#dCt(Z9-X>r_ zw^ub}j$9Dd;`H~%dwY|;y>a-~+~p)aNGHj`qh#^bqpt}5>f!9m+sguOv_0$>NK#;i z!CpK>nU)W&3m1>9NU%bB$$#XtjP&9ZLULfbl=wRV~u1fQP}5iCb;uXIiIG!XCC08aY{PW|Bpx7)OyG|Cey zWlh8>nTIm3HT;;eP@1@)ET5u*(=*$0>wD!4*g#ig#w+9N09H2RaptVceP%T0n$h?& z+y{(61(oVSy3>#^Dp6Uh8a!OU9beK*E5hZ3t(tM9=*&<+?<0U2B4wpkh|hi`ne{NR zAsx;|CgP@m?s(ID=oDaM7kqD#i&bMdanb*Yjp+qs?XH*>wBKdb?&}$cOSFSpi{LJo z;SfEeg(4!Dj_$J1SltRs@hVU&7y7H-b%6t~K=9*X8EJJHS27|d|bUOWPB=5jKM#?~~=XhG`|h$gTA@DrNsjh?`dZo@f@HPjxsTgfzU zvWO6n1EzkK#(~2I9T>wl^EpXut1ul&<7m3AheNYeAJ?lP%)T0VGjzNYSSSu`Cys6H zywDikyV+p-iz!r`H zvBSNARkZ2g7Dn~1<9?6P|Hjc6lRPx<82GmvM+~9In14${{strd8dLAqX5KX>-dmV= zdkywBZucxC%SO+qDgAEC7eeztRs`1qhGs{=(Gz>b4I|4OaEEvwFv4g_ivDJ&PY;h? zJWb2Im!XMnJS%5}f`lFCI|kU;>9*o)V#hyyZ`uWPYi1coSh{ezH@#L9Zc)~P_acC| z&e-Jg7Y1nx0!?dk@-&RI2yE+uX}i_jWqG{I^7yMqAOAkHJf1~KhTc0b(VAdlAL;92 zGOZG4V=!qa10nqNxn~1x;!fXvj=zVz$~U3@8uKp zLH&V@l+`wz_W1z4tN+hN-=EEXKezJv+2HTf@9VS9&!^wZr|aR<}h-TZ0*r= zgEvphlc(Q{r^kb*&v&QSSlHvU)9QhU6)*^}U=4J(08oQ9`Dcv5@V@sSa;&Ndx- z@UXrGofDH#>V)ShI^og~1B-jQ)!@~9KG;QrV-wz>|9MJ`UTUxl{SP$Z893J|)AI|S zlPS^#-Bc}J=jg(PfG!6fezy(;>L-mofjpt(-lhPaXuxJ@?lGsB6l)x6IR8Es-!Q0c z3B@;9l4F=|P!^~>*Nwo1PJN7_oOZ&pEQ=dvHR>!xX#U=!`>^`PwFoP&A4uPNe83T{ zTao~k4fl}V{E)Q_UQLTDy@w+S`({u5QZwUVNl3;G#Edu{-2(&@;?8l!(gs{p2! zZ2tS#9RcKmh-|*t?JI3LY$S?b5N2yGNl~V9IkHZ1>E*&h#4yJ1##@be`wO- zQmjGkACd&Jj(?pqDaGX<&ljS~(sPwn=^uXH>}~Li;|%;z(m(v{En>ff`bOg+`h(so zUm3a&*cQ#Ta266w^V)!}cK-w}AWi8A2*3U{b#&t>fw_nVrim{4v6c#DyjmIN14~8z z`vZSxpY=Xn*>I6JcnmrM@|k&<_4%0H=svbMJ$YhMcm*kQo80mmU;L*jnv5ThCzeTU zp7R{&T8&}%yVtKzLXhabMsS;;-YC`d5k83$*himDM|(j>-}#b(uGDnqX9VgGFi13- z9Z>n1AObiiuuB*%3C;^GN4V2=fAEqsGB9HceL>-KW659d5xN_AH6rKL=)C*FeF%L( z-CF7yy^Ox^MFb*v-NBmL&+Lh6vi=XzOG0uPBe+!;x!k2#qAOAq?PIX%1rfB)Bj)yR zvbmNrTN1tNfF|ZgC^h?p;r7lo9=2e&`#}{}Q2PO@TKk4x`dVrpluAW8DqlZ8bIu2* zt6t{J+I~Y|!-tM%Xr|HXD7YyK zDivXfvv5uZvmTEh2Ofz$Cd=*#2#~kEg%w(Mknz#@(PZOzoC z@LQtStrB4NMVkcM*O&kC_PHN_N0Axt#5ZpnofBpQ-4Hi}P#l43O1!{rL{5%twX`o# zJGr`QjunbL_DVe)?r|59NzWy2wTu*}y!?x@z6d}7@$0toh zd^fZIqw#p{{b?dl*#fvfaBw9tCyC9=c+QJAJg3&!-nwv4h_EK}tvSGLF53!BPpu6NOv(e(tYlOm!BH#;ZZiu6Atm?v~ z^>KbLMk#3DSm_b0n|X!*gp3$+^zdN+@uMIPuk&8Px)a`hod6ksqv$mQYwknGMpbT4 zmh@`f^)hE;uno2B{SI@_xIlkt3=jYIHJYT8M?mVWZ3)DELPx+#JzT07gyK#p_-qVcC=xWnGQ<5Ed~joKGa0ItIpD&2-hQ6Ed) zu$5%eDlryVkt@7Pt*39s%Vu&|EoG=tvkPeqjL{XNYew-TqsBOPD^$~vUhBoVpA$ty z2~lcGe&uHfT?Gz=r7?RyXei*v#JL$!nojFMB&E$>mgpme+$ zT#_8ku%sE%vPkGQ@W9#a%@}+bDh;C0Ah6Fz$<;x^CmW7?vxQEN-u@UGgWYn9uK$1b z-gUi=8%q@J-+Brh+OsWX#G5X5tUc-PsBI_Fw4JCexzjV9j8bHQWJSe7r3#d+wza?W z8t3)SlbjC=cNFSIQnKu@e%N9au&{7jSXkFBsj8e1a`ZSzv*F_(zx{6iA2*})@%Zxf z;M?zC9$%7w{MSkR{qx@VCO;pj8abI`l3*O?Sk!#^=D+{3_umJX9?ceJ7SYoqx3_83MP9#cj2cmOGVvldRYA(w#)4_fP%Zl4^M#EOw} z*riI&|E4`ze^lI@S{2}KOnnA4qwlB-#_5cV$rUnJdBAitjGXm=&ppksast^L3wLxZ z7Rt36qH*s7d2PzL`|ypVx9*nuvv_`D&Dp2fHg+L zg-f3r9yKfe_EPiN-`ov_9TQ4ABt1dV?a33 zLASTv4yS?p|))PVEZf`Am`UnYl${@^xm0pz)TJ6`a-R!RtRFJ0JZ$XIa za1ts)%$qS2GYW(($9 z9TzxBFHh4nRrs5|Fx@oeo)+I2&v|NX_AA>IdOeYQY$<(Nz@B@Jy0g5E(I#HcCSK1U zBwo+PoNiT*hEH!>w~1u*Sw%8(52sf?)t^3})Mu;1T)$glj1kb;5$em=%8?8bL!bbn zz_5*C%X9A5??>zsT_u-IT5fy{ks=F{26>LSFG4WIJ(`E3-aL5U9mQmP*_CnczuB9K zobXZ-pkl`dF&L7dkR%47IM+^#V$?-g_!nfJ%8An~ESt5{lQXdTdZD-a-4B`yk3TkB z<;z5Va;XjY8OP*0ml;7!hBnBi(25REaB4Zx8W|n#zIQq#d5>E*1MsV1@6={4HU7t$ zV?KN$dBYK=*O+^?<`DMh3JWGli(JhZ7yASKel6V;&XCEtdLYy!oH+5U_IXN@P?AxIp(<1PDG9P0 zR?ljwax^hxUAXYa4#%$Emv!JZn5n!5>xX%8DMNb|>?xHw3yKLzN5S}V7v+RAON=Qd zB6Y$jN#0r#Abe+jI@HxF^(cIFFL!(-U!J%}snDGXVP@z9`TLbF@U|OO%C1OB=ZxJZ1c%9%oEb0t3TIgqN?xqUk@B8r z(UsJvJ>sTqAQt6e7lLUIy~`O3!I;`{kn`vv!hC}UQ$5QQ0trEKJZEd;YLAbNXv*Rx zf5M~l*Jqp#s;iR}$*@y?jv-ne+b(P#&+du7UE`6>8(!m)MW5 zTS#UYWjq{}Fkfaq^O)*+kP?obKp zLW1fW`%z+Kv3hd6CI6!dHS8v9d)iWwWx{hfA{~`ku6H=-$*4be6pj|@RqrCXs+4Sa z601H!#k{JZsq&{)g-@QWs_?!%sVZOIDk;mZjQ8|}t)i5~WO{@0lrf-ilF|&u!%yU^ z2IQJ#mu&av{gm*3x)Oeu!|xjKox$Pd&Y;)}UCLT$Y_nYAYiXqR$f)>NHi+^^&&Kj! zB+>UEc3Xd!RN z44stA05>vlEn``LDk;-k9`Dd=a_f-_i<7n6y~d*+|Iy{qHM-YXMA?rqju@LViG!)_ zuHNO$ZIJ_e3jls%_1}u>E;9yM9E83ETf?A?t3t3!R+l`mTR~xDWm8a5 zjju`agz}+Yd-E@19IVh=PxO^S(|QK(y7Mby;~fjN=-<74t=-U!mLze@a4?>kT%om= z-(|L9qR3`IQ5pq1DxC}QjSFJ*Yb zhDR%Wvqb{hWxy1jtJoG=eVW(yaMuWVsx-{fIGzVb6KZCO#x>72hf#vB)qIHNY&{l# zXUStY$$t~1&U~8h?{P|vW@~4apU3kc&HGk2#|nnkQ;3;zmg2E>8{3#| zHg+V46A2<-!i*N!_Pgdx0I^*9u!*k0OKUT-+gcRwii~oITqk;tdQQ?$V6DW8wH4rR z!Jtsbf8l4rBG$Pa3t5wa=5ck7m~beO7obLonbHy#E2qk)PM=*j8H$~p%(>n`vKf9X z6c19uf*NtXkOEmeLIO5;j1Bp4D{im0s=VjgQ7&SIgWv!mEI*woxpQh-zX{JL;nO0l zLtqGG2EZJRf*i=VjDIS&+YHBZBd^u2+U+pd&~^~##IX6a2i^#5Ytsugx3VKYMDe>E z=ilRaF2nWOXPY~to~0y&n$8O1L;c}aEl)-g9B5@&)h#q1EGrYLrj4HE2JdCSSst{l zBeauUFu>}tRbjvC?wM<{WGR+}q8E2t*9b#gmrCQ#mqwj8@v8pX6>h%EkEJz;8Husp zt@e=Bk4|x;GI&Q$e^%b6w)xA0-LQ7PF7H0|_oQ~m+VOr?-LVd|di?4S2^YcE$3jWN ztDOtg$E6dasW5V$0a@bsbVp{zDq-vZR zuxkcVtyh<)!Ic-(^6aca#vMbA<}&=O2D{>3tU}{c?Zb?W9xg`<*!g##iMMCtS*EO~ z9L&mW>&UUoH*simFB&biYOH2H;r3#96m}MSx!fgjgje2A{>DGiR^o;lT!8Ulca({w zpW*cMqHcfLKv&yl7OXnN>erc#T6^xv*XXC)f+jldXqw-a*pL?JRe8=zR8||l@5wca zNiya3Q;@33vZ<7642lzO6gV7&+I&|SNaWGKKygu3liMPgJ_=ynLKd`3CS8(+IO~fa zBeF=sTv^3(b66?0G^CxvTwH3OQK(aoA=kUAk%5A(cej6mMG-(|e@N2;vY zM|1pBl!WB^2n}+_)(VZuT*@(|hKDQ8f*B4%$PCU6GtZJBP3gi_trto=xZV9`wzwRN@OA;L(PIrD1;zA0G}Akp+(ZvUIn=MBBl{f zZ#25$KtYNQmQjRSDV{)MBj~MTCan`SsY&?9>8rOVZ(jXye*F5#G$-Uu@aWq#i$)8I zl}8ClD7i~)d5QxtbIQj71Y!*t^%9U&o`MYX;f1&n3@Mc1xp2Fa>b86rMIQgTIk__0 zJ4ql9J*!vQ29Cf3ZnOZSL#daiQL>PystVtu&D%hmq0rGf2qH8B)0m6^Cw6o@(anP- zl&@Ifh>ck$vp2vt;fPU!0yM@6&4L)kghyIZO`WkGa3=_E>@0kSAa`Iw<+l6v;zGo? zytw#Po}_b3gDwz8TlfIOWUi#wWQt;Zg=5#dafFS0GAObFlIgVmQ8FPGEXZP4h;~*3 z>|3aG#QvPjslk?mC!kc$^2ACw<3YLdUDJHgP>=G@Y7~WSqF~M5N7A7y!+av%y?t#f z!HiG{M}{Nq84Z$Z>E z)ok#ZsiAFC_hJFnF#3p{BiS|Pk?$|R&bwuXKBpu^K|-Uh%1=8E5|n0`^-yr4TyvfA zJ(- z!G?%b=io$qBdD18=H7kRfSg!9ttWC!@DVS?g zFq_+hAJP|K;mH{B$_Os^rRwKMF`SgXpTByGgV3@BTx2mv)vZ{8{Z zVAo-Uy3nBc`tek;>?Vw1<45WP8#h4888|S~s~x#?kk0z5689BkjP)6bSL*({|nLU()lQh{KNJfbMmr7}9-#=vX zLvcm4Qy1zah5Lhn&TuI584d?Jp`n{EQP+HSSg82ya4`4S7oan4HxFhyFc>(?!7+#; zlNY+Hg{3Xy^1gIXnA`F(ji4!1c?Z{&Q-v*0M)x8PqPbc7kqqeoU(Uf%V=O~}c~WD<=dyGNn9@s%I*`owR1 z#4h;T9y)f$5bPTj&SI2ybp}rG6VQqBt$6eiq`HDg6FoyJ!v%8SVj#W8=E055iNme| zc^*OZnH$+OC1{l4;1Vs;X%>Vycbt^a2{aajSCP7>Uq$%ZzNqo6ZamyJIJW7y`3XT$ zGA8phpplVoqc|97N+K@T

!Ka4TIH8;)hKZe5E@gNy(fXw%k=|Db~QhA5}t&}@+^ z&h_{fy^3MM6MDiuX%ZzN;+K7%>uy&EiKH`pqZcYfKIwfm&uQRyMzgTfJjM_!Cw*UD*NB{emf8ROU{y8*oaD#1p7~USSrCo`nw}W;p5af@M zqRXy6->3n`xXC}bcPTn2J6a=I3giwsXA zl6>@hA-V5ukd0@)_fn#9L2uF~?f#1-c*irK(9+n|IY}r{ zFk=0ZM68sYev}Q^XpoSklWIRPGYZM)QgL+&I-|hjQHCf!8BFg1DQ1krieaf}v(zH7>op1Q@%T85Ajvt{H z5F^z|x8k4BsG1nJ~Mp^83_^&nEFNci zVT4_Kvirn@CbA&9;(lRNV+N`)S9ZziU~SRm&WK%hxwA94F>vapJHvC8EiAs#rU0%L z-6uF^JhNZFK0HT`=WLNbcx2pSa3zPhikO7u@*#6=vkFe2KrI_OuJ~F)vJ9cb(!!#K z%@zenKFN_ps9u1y#)ANc)x8XuB-uCacSDm)h_RMd+zCx?MvSddkQm(oQMPx4lcA?P zB&DA83!QNgH3~qf??x50vTTX}L5QV2EA{i%j95wpWB~6MB%$~{MKcnIU_aC4##Abt zctC|29=FFCllhz^IxeFM4`!P*ghpJ$gd`{p4 zQ_Ve)+XJ8?Q!SF?{JV@S(j!31eEkDS-V#EEy{GZvUCua*p)Nl0fXMNf?d{WyfS65b zOL#2cw>+krh{R(k4kH570O>IhH@ zV21_G3$Y?ZY(&^{0wzwbqKqV#J?XizSHM!{eTuUP<|O9Uk-?3OAHli>Sqc&w-@qCn z3(9t3hPhE8qnMD`IsxNZK}QP#49zxRan-gl3dWb$K?W-u2rno0CGFg?zMm&Hb7La# zx7N<)q~(;0$upR4Gy9E@pJzvCI!?{^D)OM6D@Vo6_-MW6ayc$o6st+dTk`(n)3-nV z=c^az7bh>3fS8^9{BLvnV3df(r~~eiA^@}BkAS4(_fZZJFyyfjO$?*3T_2F1pzO+8SI}_-wuI`T$_8yV zf*Tq{+J5^;<`hjrocXYk=M0!mavCl3MnqKD2g4~dx=|#6TX)I1-L;yFxVaP;o7gdu zdwePOd}Ctw7{T7B7GSw=1dxdIX_SgzhA~3_QNC}~%gaWP6V9`J=k>TS&-~xfVtmQq z6owqB81CMWT(0c>#H|u8igRXSu09J2P2Q_Jixm^GCMseajj|xSadh<%4{_MrJLnC% zL7YZL-fWdbS>17>aorp%&C_B93p&FI6)AN^%4RSEKedc- z6}?p9t-aZu7ikzM`+i7d3fuuNJ1!=H*0>%aMXEgZ3_7hmdxC6gnCx_sJ%U}-ofw%XJ;KA))b;e!&;qYjVjb|E;T3vL9S-E>e7-QIrqphVr3 zF0AiwdH}qq*9f}3J!)N@22&*E;YbL>t-(=4?Bv|%k7Z{fWh??xMkpDR_z0c9I92a0 za}A>z2zzwOL&MwiQD$mVQu!*#`r}|c!zExmHZdifA9A_5q5DDbbVDN_?nx5Sh8r!i zik5D+5bkt2nPbNh2yLf0n@8LNbr*$EhG>TI4kC$)A|*|UM7(Vk+{v+Rw^6Y(GLj6^ zH<_l(B@tayQS=CP*jngbD6^9;Y9VQ_7+c~a#J zs`C6G5vTwzVj?PTAe#VQLJgA;-Ge%X?>*^g;)^SfaBtcZF(45ompI|jcu#}!Ubs@G zAGt3F{+=Wx6AlcJZkfl<%obR>dtg{6{5E7bmCRU0p}vJ&L>44CO&O14N6%25wt@qQ z26y>&RX1xk!3seScuz?Y{mLGmM{}IhV4j-F8s$=ilL^Vjm^Ubcj*H?^Vz|K~OZY?l zEx3sM#26FKr4*bm)ljK1NZca)s&5CG*mYx{Fg}XAaC2TqI}8Z+Xx9)V-0+ViiEotC zs`6T6N1B^1O|8i02&neh2RE*c!Y#>Z5XagsOy#W-+eka&N^oWd0%JXt4X@+u_=-<- zs0zZ;k@KG`*1_mHm+6oRB3U$z5}j_13w_|y#*@(SDEqnCDUPq+`AmZ14k5x9{5rS+ zZ!Pa|iyMQ4JHiY%7x2qs41D;6yUcHBi{NcA5nTvr!T@`=Ow1PH*j$u)W zuTtJ9Xe1|cFrA^)l9!)IcA4{47)?s>HTOmjy-|YF5ur0X5*-i4DBkc;oK|wYAj6Bg=_B zZ%FGaW~O3sFF!R}PKWPM-oX2*28^$B|2P*TO}-}6Z{d&?DurC1yDIKA8`rhJkttos}x}-$u+nQRw1Vu4(6Ke zgSDXDcB4ckQPV0J6ULvvL3_R7)82q58k;lzF2pp5;(Qlud)$nLWSpbz3Z0E`0sv>= zZcL_qZp8GvF||(C;dePteUGU)=|w<*3!YofVHJ5#gE8efIQXEL$qYRa$4J(P-Kh4g zOl@$+XG)k^m_{2OOnQtRQgbVOz9~`cg*8n^k_RgbqSbSG%>5PP^kqiU6o))FVs0l# zEF?vgOhCvhu`@+y4YPCZUE-AL>=m-(+T+pS9+Ml`biK<0I?(OWO_9sGp@DD^O$c|u=~<_0v4)u#P>#PZ7p7PMOXtDl z>(m}yqaBF?8`u;#Sg}G4sIGN}^bNA#2(=$)2iR&cJ4FfDdjdK^tdxF6pLk#TBEI|r zgo1oM=MlR`h#jD*Yc#Q108)F+Cz6`H6JweJcK;D)v^_s(WqU(Ujz3$dJ3W<2L%ow> z!j&S$N007kGR(&0WB}_}HX(9{H=79)yqt;M+Ns!OhcZs7AypG0A$X-IhaI`g$Ok`n z$AK^^N=VX$-D-n}q_yml34EWSx8!i-9_0?se0uHmaBk*q73)EQysklOPHL=|l**a;IDZd9b*diuXkb|!^8=3;Os!x6@+xO={l>_(M z$SUguPh4we*^>zh1Um|KV6`PvNx_ZL{EWt}km`1c5p16B))+#y&)hL|2*N#&p=`Y- z|E@KL#RR;Lxl|mJ>mRRh76)k>CDXseWPF*clvGzZqtO@?2Qn`uG^gJ4=rVJhah+U& zBaCvCQ`UJ?SF0$SwL)^@sELJEc9D+BoH0*yCaW!~;i%{#2MQrVennJ~K7UVPO?hGf1Y@SD)$fFRmBSP{2JxdNVK-;WIBSb}^ z0@!j9p5RV6Nino`_)yHF{QF`;QxJ?1XBIog)_1CM_? zL$CgNwj=WJa|PmyA5L_MaS%sivLo(mXoPuGKnM;);%*uT307^5ZUEw#+rj9(I3ib9 zO2YgJIyprzPF}v{ZepR3eWPnUnh|oD%VZVknq-$Mt0cdr%rKgfoIc4>f+_oM+|i>+ zLHa=nKfiOzni94<-2Nq+gJ}~tJ&GolMUhCtBY9aNnPd(BAX?P{LWD3D=9KZ0VUqLY z`3%qtKHT3%Sx$e7=h6TcYTc&w2hsFXAvk&dc#4F{bShb zDikd1+Aa!)y&z3{>?$7HUqdqRM(0I(MUAbQp%t%*w z6(nPZ7*~yoXld*|Zkk4e%2KHu7?XP)<(SJA%XOS?-J*c}tZV(#0ooR);Z9Evz6v@w!U+Gq2x3OEo(=}; zj;ain+-9H!|Y)3NZ{|U;ZBkv;61of21U%e^#3E3GlN}Aj;hgm8O15 zsC?`ol8;!6MU9IdJ(>kcK9RxJ3O-^Syb#gyJG%|u@OHH|;tiXC&}9|yQzGxo#U`Zq zU+O18e93)Jv?v|e2Ukg$rg3C@zCZFuE-}LH0`@XCu4sj;cVgr8m(U98Q)OB82)%)* z%Zvk7rPN3H2m5R>D$-;+j>ZvHx*N@&3BH5~e2M@e8W;4(2}C3^OUfmw0t*|nKT?(i zZa;A$F=b(^%OX$of>NeQd^{GEhxr5@N*-MaZdMyOsZMl-AF5o-Mq`;NU9y`M!7-b) zn%eDYB%D@3i6ag%&MO+m5QfZ*=L?e|8mJ*4wx-9QQByakGP`}puPYWvZ1E|8*L)**x)f~L_l^xrQC!_mT+KkUoz7UM)utY%kn(akj`J9IsBA> z=M1JX%|7TlbE$+a;Gc>PGXO7+fg!gFm_N`saHqq8!b3SgPzf-Urnn9w&S(kDv&$&8 zAYI{%|0&}^dm%=U-9N*u`rla%mw5P5zQi{*s9xfmGEBKSI~+YSR4X8R1D6z?IMWN4 z8^afD?cczna(vAw0g4S*KB<)8T%249yAXR*u!%g(c2t)4@-D*70lLO*MUlt3*`FwJ z+=RTbDukQR6Lv^mY;(iP9Mg~5x8`F?!jFpbe`Fln$6N$|$}!#P{kwL@S} zySVtp2nJU>*3vA_+{d4ERI`%tEOgcl<&l!m7tW)~UF(0gdtdE5R&NW@J*-fbbq-uI zT8PCj8;N%Oys~Ygwl+Tbw2-6cu6g8B65cO=oRUx-5UrVH4|^%ORfDxOCZ{=Dd1`aD zibc-lNDGah5weMmoeJ^GS7EP)RW@%Y9u06>sc&kg1$O#sYC8utOmwPJ6@RG=v|KG7 z?)fgIXe1P@nZ+8J%Y>h@xt&ygOC|j^gST=wledc)0JK0$zh_%6&9@y=#$=w71Sd4_ zXNwp-QnnsF+A;tD7sl=0Vsc}F7Wg@lQFw7P!U;p^yjU+N1_&{(3&9rWuwVg~g@JHs7}-Gl}}2zd^&^ z@N1Cviff!0WEcnmuq$*yY|UOpfv^|EaVQHYzg5wSDWYmm$%Gmf2Ea3?+X>Z&DQ@P4?snoiNrx9cE{oj~i=+RY4-)Qu7mY z?L86+{fnt&B;|2^3(T3Ku^Y~T!QxBUnCLDKJQ889CxmJ8CqIx7GaygQ<`VJp2<;D^ z50t{?wc!6wPLYUSiE}MlbBuz8=f0F)M!7a9(AVhd16BtR?@Q@%|B=jI(~VMfYw+CR zz}pZkaZ_BiM(j&vW=49JEuEa|l^v5H{7VoA$yhBB_w^ZEsNaG=oyGLN#5@#BG_eHi z!iW!$1uCc-W>=iaBpUO`79s9}J{plsF4FFG6l!Unb%~NN&N&FK|38x2|AGdFtp{tt z0eyKP_d@B$1Ioa!p^m;l-k9sRrl@gbTu-?>6(>8|GtUNXa%)J3+K49-p6xy)qH9Z; zZjaxBsn%Lr8Oww#lU>Jq5!OQDDq(_Iqgo|)50}o8E2rP6wwX0;gI1|&#yMEO1!t}C zUx~$aRz{bTwiC?VVf z{*{mY(r5gug0f$0$^!5*WL!TDGpM>7@ ziuR`zUpl|Ga~~)Izgxptopak%MlEt3myuH>(+~a3r+2E!VQ;Mcp0f{n4ETf1Uhnv! zE6YY-B73tVbqVaQgu1meXw+w)nJt*XNULFbB#M;bOeLe1u{I+o^iH1Vqb+7*(e<;T(*#EisP=eDFjwT z9J@bSyaJgfUXv-}Ig&-acR3cxlF_!f*zd6E`YmVIBQl*zzNkF8vYq(ZQHR<2npyAd zYmh(x^I7ySEE9%f8Prv#($~Uq-ii?SaHDNQu>so}vH**e-@&B_2de=omQN%$60o#& zbpeB(VAYDL)X$?iR*Cj)$<8d`#^XVh@aTXOk||FkkTmWc%%1G>m@A8%8iXjr(?uMx zE05d^vb}!}y3c?4_w9N9G5?yF2^fPxJ`~LMkqEqPiPY5l6Q9iIOa_O{rPII2CdVdXMy^%#NYgP^x;L0( zSxlt<$ul$>`EV;!F1i^PnOi86Z8jcRIT^(A^No>( z!;$d;Tuv^8VC@K9?HS(?AaY>`8x^8-$xFLw=yq9Fqm8w2Wi?)#=+;FOJV&z3j;&{oUKw z!pI16_+g>hF?TH&nw4%c?_zgFj8-rTc6lOTX9AJ?!E(R}%f|MT53FfYSGBtrSzNNU zorUr)i;t`f(n>+}?(J*i_(0D2p#{8(7J=)j_Kr?4qrE`}G9#H10*y(QF+PutyRU3C zaT?5pCMCO)A*&z=JBV~&{%{7Qp(k=f8df5Xs)}p>?VfUlSPEaPLz2ff#aYf}yt5z) zWA+Q!4vF$Y`v!6$>MPL{8#nOu6%uFgX}-=PAeR6T8W!=$#R_5fgRKLQngw=i(j z@{Kq>s*$}hSac8%334M6l1b?0MFj@GL}@Ic|)7A)E!@D z(;5};c(aOqYBYF9`_t^wx;Q@`(=JG(s{Pr0$Q<0ex35=V*M!_$PY^Zv6D|fygD8XL zZ5-e|bgaxQtdu^&3l{y#$NmB~vCR7Ot7d6<_|+f`RYq#4CiaAILYE+gOeH1ZlUygs zF?`Ks&^#ecrj< zEg(SLl2?MZbqoPp|K4~yxy+MeWjPWwB=_3oD4XoN7C02uNM@&uoI}~vHACgwAj5M) zahI`bxM%1W`3i+OKSCe7Tjmn)@Jbg@sbgnr=%Nk-+&N;U#yGmdS$E)O5Rm}H24IEv z2YI2?B8xjBK)E6vT@AJ8ygB-M@b!?5NdIcs7cbqw5aJO0frKJP#Hn%s8P9OIh(Q(L zmlIVR)PX%27lyEjF{goj@@hf^PFmEf7+HwV7|kQGzC?1YQ$sV>1~*1$7Zi6@Lw#GM zC&fToC{>gJ@z9n>=wL8-6IsqwK|-T$Fahb@4TNV+GmxQjPF;+v@VrlB9@@pqtHuSR zDsd|#Cv42Dc+pnOPt?iDgmy%%Efv~BwZkH7!g>PE>w6`vbQo zH^Nj2M#a^<9w#(97L_a>^km{d)F$wZVDjxnNlIcez45#Cq0Ss+Sw zO|zk0B$9e}h2{JR)1YUs{(q4KS9IcBqjZGqnli4DrvH$$OL9TZ*hx1ykEFTj9vzc} z9ZtEEWRa%|1MeO-@z!n))Lf{|%F3uIQX5vTwU>|>?2Iq8D4Cvc4vxQm^&W)uBr%1( zCcQex+1yd3RVmt(s?F{!-oxGCUtP;Wgg>Raw+ihW02`LP-?NyCf+yw^Hg7S`Y<0x4 z6U7dB+6|JhtASdxWwd3Pt8kY665r_b=6WPLuYO+yvHl`X;ST(-Ut=&_IYONuh?rg6 zY0!DT`ca-{IDm^lPBYexB?jY~WS21s!o1H&7W=0w;deRwt^wZ}7+&rSioMXKtYMl5 z@6Rsrwe)y;WK@LhD}R(y0EFPC@-E2(8s(FSdrG||KM*R%Oceyaw5T~P)tw>~q^b19 z=yK*u4&g&g(TKjf8um^}=;CGx5La|Tl>LHN_&v4e{PRfL_!>MWd_nKNPaSgjHEXTeyMy4g8Oo=2IL$aXhf}%5F-g$XB(wFW8@VVRPyFE4}CI#ZVV(ovZL3!6t%`KV+nDlnFzSs(;TW5ckV5|FH8 zmF*PdS`c8CJAMhN5r?w9;9>Ag>vIKR7UVM=a+wjVV>Hfi$QE-D=Q~@7!xNG?*|W&- zbOGUq5~3^&wT}$NS?N!WM2dUy1H{1#I^)?q`8hKo)|AWE6{eN({|MXA8gQw392M5m%pAG&r7(PAR+y4_9tgDk% z&w?ERe;TaZmfN|{bh)vZo63WTA-1{tI)|jZsI$KK}+Vd@BN>3miLJ zTYr!8h&B}$JI#xb0XfYUT7o_!*E(tw`d@~2!L<2svS}H14_w*LsJA4jz0ZK;RPO(a z8IIXe)T3#{A8QAo67F8{>na^S1>433owW*@ z2>CzMcCr-Sp8aK&&c&EE>6%eYy`KeHh!Y(Cwd8xTf9u_kzmX9qfF1_K-}dEx-sM8* zE$hIAtAKWA>PiFp752uOkyGmMWtZ=o@!=^ZmRfD=#U#eG5gM{}JAzM7isnuOI@1_E zW~F%)8X>1*N4>P9u#U(cTm7O?@4HBqH#d(3NoJHsy7vj8XdC}t*l-oNqKnQvtZcS5glm-P>vlNT!_y#zrxuIGEajHqt2hd8TR%Dy+L=-@r@2^uYiy}S>^aP znm=(Tb-N(FsJyF5)On-We^ngW<5N-aBckNLf_ZF>k6b3A-b>}}43~$Pkjq7iIzgJ| z{i|UIb##Fa<67CBrps>@9o4hWjk;}XN)xnZCC+7;AW{dM`N8dL0X@Is)gj(gvhT9);fZ(U>#zZ9^`!aNb>@5 z>x@XBfN`C)BY?W>h1Y_>XNgdHB{k6SE@5Z;{l~C3JYajT5$$1cJoY^#KjMQ^q@tyT zd{;};vtG%y_{^iLXnw?V=0`E6H!1#RzQ`d-`4JNLxBge`@*q7}ObrLDMk8Y)+?i$I zfv)L^w5zgc1PLoRqFFGeXt>XfZ#B)UyDN})nFwRHUEfc zwn)b4Z~%Ym8-KUsp=nl8>Ky#Yn@lJ2+WFpA`2O>1hIiDI4gq_-s!(j8w zkQdl;*Klc|&JCF(2|-5E(+XZ~#0drfST7AZi!%NMw9b9Ea^8$Aq2r!Gm=qPCI1IUU z2-j`yKLaF!F}t>MlLJ1gko{h{;62^%?jL@$_q10OIQUnLl2Z82Ao)=(6j`RUfLuv? zPb*fE+`MoMLH8{NrDm}jc&ZuZP;8;P!qs})8bK}L>;7BbaiH;b8^QDtqk_x}eJjS( zB_EPF8)WF18)7_X8)NUOxf;j>RnK)(Y9d>$ULZN+6=If9&wzN4sgA8+m*c4_tnP{!uYituuj(u@X~zrv<@Bk zD3S6t<>^)151y#c!?X4@@Q%GUP)8JGO!DQ|z%nov*LTF(I4vz&lgl zN!NRB*d^|X4e{fpfrtfZ>fAnr_gQyz19w!#1jxY8-b#{4*j0qW-&0=2t>}FCP&YWY zw;d0JG{|U|OpcnGDhR@$FW+^Q^dv$>b?YvgG<-zAF9?O6&Ttk{sf&AD7(1h+O9;U( zn#&Bh%tMGXdEY%N1{jS2HBmU?t3%8UPzV8WB`RacX>FV!g3Jn@3(&X0@dGEYdPHeGn zI>_Oj&HB=fwpkLXohxDabzrcQt>?Y%e6PS{`=)u_Gq|?NF`2~`r^jNR8%|I~owZMq z#TV~1LSeaSY7DW>MA2e;i6!UBWH%d5Q+bsY&b}g(-ShLrYnha$085RXAm)IGS}!#GuwR*+hb$$S*CI}31Zc~fbiVZUk_h?#;o_-;vdT$8V| zQl2q&#B;naO{~M-2`2+#ZDkA>k&WKjFmozZ?W_uyZe{zI9!?$}Nu~A}FEWS*Yb@le zcB5?cPWbP@QDXVuXm1MbwPAcVK-j(;Mi~-^wx@>iEy363w|(=QJOYjH2lh(1jFRvq zN1Z5v40Ih`$KoAalPwGDR!cw(hkqSFXUvAJC190gLt>S*r{2GGROcEHf4yE9H&|jr z&A39GfSm=npF=DYod-csNnn0~=h0vqVl~QAIAD`O;C=?r55{LHr+OdiIx{o-<4f317`nhFm5ld>M;# zV1Ms%0Fdf3*SrrX+LSO;bmcr<#4!($ zlJ|r(2}e*(M$kZHf5+MbgP%h*^m9M46C z-WJE>XiFq}Gf-jgWRO$c=+OIiZ)=NWTeZMkFeZSeEU^RFFo6JXOP{N(Q5=~kI%9b0 zO^Sb<024b8(tq*+`vp=cPasgiEtoNMk)bgcHgw27L!lHO7`bNj!CAS0+N)KiezO-Q z5H?5~j6GI{i9q98>pqcbOUh*w^?;mE4GtpQ+`#jzY!e zFGr_1b)VWL|6ZTU_z$M?%=*#03FgfsLR80p*xNtccj7$z#DsmbzKRco43_dEx4kZHb7`FZB;&>1-w0;78gtl9~+qT4M)OeosbE2 z(e3R~%gPM-+uQ#;p9On|Puo`JJ?Xeo=!Y^EpR8AWP#@F`cN`Y;ZgA_^tSUmves(FO zi`x5EYbu`wts7RA{cM|=J+v`o6<41h4rWN$#xRUpqH_g9p7zI{&lpG?VZnkD2xod` zl{qJuxXo-BeC z5_I`>-WMS9eraLV2&*Epjp*FyX1${I&h2eqEdIO?v(aGd*tc7YODmyXa)w4^bQ>Mm!qG30eVXi>kEan&h^ z8t*e+L6CQyYr8pdmvxlmg@PqAY`201bsSJey66_?mZ)RFYBgoKDK5*U46p6PNhqar z)(R&cb5lgK2E@!aEyl+Mv>HN?DbGNM{cb;02CVgrR#3lr(lX_@25e=JG%Bq$GQTMoCBgl~kbSa5p! zMf>qHq1BC@5&f_hsF@?J>6VqnIZaS0$d)EP2E5kfL)NVu6C*%fyDVkMSw)%h%dI7JSboZRn#t-Z=Q$>qnDc8KO=pn)cc6xewIOE{XLEB#?sAD# zERwoqau*Bb6D$>Z3%4@%l}Hw>h|qi81!YmKA>b3KR_@{S3% zs+f-K!-o|hF3rZMsDP(f4W#pOiTjZQbv;8VbH;JCnaG~k8aD}Loj8d!eFkD81K*O^ z33yckSe1mX#px=fF5=ibpzg;dkoACH7UK!Y#`qOz zAYX9kZ%8Oeum4*Me9M6^i|#y1-r^vg7>?*dh5oi>5GxqQN^!5staXWRcERn6EwpBA$phYDfG3R8Rk(4xSe1|KQov4gJ55r_Qjt zku{y=m(xX$RrrLmJu54!g4#yBb23k3JjV&AaFBy;5{USjBs{Yls_ck~)JJc)$`z5@ar=KtczVMoXhGQ;GknXIOda&5n)K)!#M3t}qo`Hu zB5$-Pasyop358jZMkgZB1%tR+d<+d%6gqRlakOlr7|p|O%4BdFNL+m(IoP0b#z+J- z@pDGjcDZ9_N8wq?=*S=hwMu6XxE;SH;Z9hdCOwUfSjv^`n^U1e2DmKe#GQI{j#_hn z)SBz7t-=3yWXA5>a*04S9BZm<%l$=+^P`4H8n)X8qsZRYk`|*XLS4ju@->pi0>WdH zIpk_KF#FZYM@F5F?6yod?GsT47|gRob{S zo*znM?G&-=ZdTJ~HJOcjX=2$*s-uiFmKx|dZ)VtK)-2d^l?Mg-)VG$+v#zK0{XfN- zaGGl@38=aMA3l9HaPI#H!-K=k{r^6mmG1wNn&IrNSQ1Ud1)kz<1qTKQp9n!;<#|l@ zGHAt1t8@otw2+$e?)p2o{{k%#<`tTE>kGW+Dnb71C@&G?Hx`dPc5iFl0(DyXrW;W_ z=clOP%c5QM7zSvO&?uH`3(mvh-LPEU0<= zM#c@iQ4+2KhAPTo(;M+)&B1)Fk&+CkwGq7+cYl#Gm=+7{Z=@gxfkAO)A;=>SDFg5d0?!_=Fraa5@5os%lqi5d#wFcpguPetMYWuJ{(>*bJctsTLAP&W z$h?h7%&Z%k4b^i$L(T4JXw`cfYJ6Aoj<9}8?rgPd#(YZeZs=|Zl`{+_7e3VH(uXQ9 ze#mqAyPNJbkv&vwp{~29N#-tE8s}$PLtggL&!BP8G%jq;qD#d3_o|;tHNu#Wtn9Rp zlK8-+1#GwK7E8lMijsmVqIlOg@aFfwl!Py%JX@sf9Qw;5oHn)tsdiOhI-h2+ z3Uba?s0HueC4qyS^{yp)9(Wdlx#i;pzFn`a%Z=ZS%Q1rzMU7W0hCjRm3#AtHKTo?E ziE(=apgQ{hY;fS({|pbGZRr1fJWc8Uw&5mi$9hWhRF^X0h+^3&7|YCrzE5eMa+`J$ zkxCMWh6yNQllm*F%116V(n{NdTd&9%t0D|uqF@Yf6K^lFRkwuyxp?^N% zYvsu=R%|b)jLwQA%ju|sGWZ363T12tfEr;|9)c;3@*|-4|9rB@F}##DA)X5lkG|qs zTEDewd-^|$Aa_KIW1yD*AMCmDpAQEc`hPFa7ft^g6McP9{DrUX%`s4V43HSV#S?~x zB%l?8yM>T(2(M`<^Tp^lJeCNSue;E32SI~kbyV_mQ{W~;DcE<6zrH5HX^*gjsPP6DLit!YOM`#kn zIXf2~i4?eFl3@|a0MVNi=RGsz0#`3QQGqm{$TxVN(wmo2re35OVMhcG)f+CIS3g5? zom>Z5czk-IE3vhbQ=HJhLyIrsAeytUGm_IEc>YU`GJAP)`oey3dh$}g$QH?Q{w~Kt zjPwYd;tXQEDmn`iboL5hmAAD>rDsko zE>33AH;bG?aXy})I#u#AIln;&dFLQ=KkMq0!Re`JPxB}_LORgXJa~VEo*o|VA8sKu z$vLxepKWG12!BlCo416}Z=)FJH#x;~Il~X}Ja79nKmV6<{f4LJ{NEoI^gqLAgT2l9 ze;>~WGY4)31F(a(6VlZ8Ymwtu@1vYXi3q_g4)*?6oI*{a7^5g>r(qo75M9r30{fH4 zE{jOQzRqwwM^Tc~AdUe+R)Xgt6uMRv5}ZT0e9W3Na-=&R>f0qQ>h&&peqp@DsgR71 z&2~)4Lqc*pA-FfR!^Cf++ckgvJK))4k$hB;q#GJ)aF5zQ8O$K`w)dC;2! zwgxvbkDXJ8&1Wy4^=E`!c8*Y*;jWn7UHGv`r&$oy2OIzvZcPPoDSd@TD&s3Yr?RRVv2};-HvuSggMEq@&;k!stDM>+N_jjklQXw!UjFrMD`GBmDCsTrx`L0fp8Jh6qw08RJnk`2{ zCZ{?%byAs~oHic`6Ua*Q5|beOOArUiSPz8#u6{I%Fjhd~3~mN*K_*m0$NcIzV2`6P zt9A3LZb_O!11Q(Ntk|cdBwU$njB&!K^Fpqbjy>JySumMIV=lq2A7pQ}!W{-Y5X=Pe zOT~LN6R|hw;SVGPYj+h+J`peKdnHSHtoekSKrM*Wu(RiKhGtFcvZ}p|P%1+pY-hPyfI{g2It&CLi*IePGiOy0txu&K%5A9A@c8+}pAz2$7 zhjIa|<7(!EReCR3E)}-hQoBUh6$w@DDxJ#S-5HOyrvIYg`sn{~FgS4KfBOd;`QQCK z>!ANW8I*Gf=;hTLlMIu*HzxDGdo+m3U%HTQHCkMr&cK-ted&5_$mzO@2vxzqE(NOn>C@Co88K1Bw`m`9OOdJ-FwS_^+c>*C)lGceC45R%;j@d zL|M6aD6x+Y``cFkZ{dTJFeN;17zCE66h_pDVHh;FlfS1l-D@mWHtHZF+F=blAZtff zz7~X{+7vA{06c*g8=?Zq{oiG=A)tZZjZtv{ERRY)o?$&J{8w`^zg2m8Bt|&Ma0V>S zw~-F*;k<5)RtdP}@uP8WS{p#z2_d;aCJk!^O4FaGB^dJ9EgeR2vS@y`m`tMgsM9YY zZ^iIoR?O@E9Mdcs=N%ctDT~H$F_88snHrIh8;(f~DPIP0v+?D(OjcYN#;exbCi5>1FP>U=(vB~AB@Z|xqNwR3E_LGVif}fqPL1jV zNQIYQ4W@s>n&PUCmFg3!6|UtlSHo4DJ>>^jD=hucR>ROehT0r+t&nfp&}!Y$Cs*xx z*a~|u+>eqtG(9C@mj-mUWao+pQ27B@JUjK7*Q)1@8ug7t)rr`u+a_6UM6z4;uM~RY z+5HlH8a7Y!r?vcV{qdjopFMTszZ?wrH}b!GdDbESs|e`4-uTNLMxAMn6Gi#{Vh|U0 z$ogUscN+yEcDl-lyKOy*{;*A+(CT8RxJ{2Jkf%d0ksG3<-yYXfv_5V?JaX@^)} z{zF+@W4mUZ>07m-9YS7XyVh+KP3%g$;OkR@-6s7;b|5+}HMDa;l$zF+;G&G|WQVjr z)^b8#tBd$|e-{f@c4F6>wsuZM+q%wdrx02=Edh}hsyjAzF0KWuI#*$U+6CfUmywti z&^f08#gj#xW9s0MlCWFErCU#kN*i^_tTv$XO|xS%Rl?0z@_+a}n>3u|{V~aKH%#)S zyg=&ge+~-qKlX=*oA|Hy@_6_^HrFqcyukX&_sWjg`L370t# z86cN#+cet|XuFkVVZUv0X=!(ab%<{9^YZJw3u^!dTu!T?iGO5S$CC8Fl@(A8{XaM; z#Q!@ue7cGMbT7~H^xw7-Sm7ABf7>8U+8uJFD;j?Sqa95XlJQ8MrWr_HNzvU*0jnAz ztux4gJqDIhQOQL6?1>~5Ewd?d1j?(1j?(9F{rM-$YBbteGWo&Q88KC#Z-%9uI_7vS zuL8fWwlUbQg2Kp7u^ziBlO1YfQe~~#7G_i(V;Lt1@N^^pxtGVo|FgM1 zPNOrJ>Cch957LM)(>r1Cg+a!HX71e)u-JQhejgr_-=0;k>QRwlcR0zlaYfEjs~qM= zrtz&HRP&mZpc-+Utsr=V5X>k|bIWE>ziGx%R^s60D@(YCmVnecU6*BaGbXK$yS!)q zaviuEEasgsnUxc}B&*pNW?`8)(83RGul)fJ$I|KA(zAG-Yi;oe66e=ko7|NlZw^_vnyz_L=(qhocg zu<{OB%cY=X1+UI)mjiayj-Enyp+;Veax9r!aW4BTubNJzaqLHYaC)T{O_n{AM^)`_ zt5R{z+j#7)`I4UiC*mXmo%7K*XgJtA+-n8ltWf`%Szig8Yv#(|J#|pR-KB%J<0z-7 z*P>5Nac?xtFuprh_bg@qMl|+EdR9FD8<~UEo&V2ptz2$aJb1R9g^VX^>HkNVWN4!@9M` zh(?UID#GElPLx*P%1vQKer-MHf3+U4=KO!UKiGH9|7VBKHu}GNdG6-?|BP(|%Xj1- z*H*B4ObW`y+GX(ol#HcP>RRMiwJU7WNJ}fjMtC>0Icx{(J}nTd&dP^qk=UA-US?Js zE0SA7x+~+sWqR3IL){osrPdt(UK_Tr9b*moLLoa}J8VHgj&^oLAVP^|QGb3Hs>Qn+ zj3kPe7>c60u`n!tRVFLqmsT%JM~f=d3N7qX@I1@X_*rQz(W>pXwzx|IUAtxO#(ttT z_RZ}lYA1LFJBn7YHwL_S|FpUP-m+(GlGIZ6r`bF-7L2y{UM0g@E*L}Wb+dw$ zaT6Gz@zdu1hv_(M=J8`cb@IQxV*JOahl9=i?>?TA``;@%4!svZ@!>&x`LK1mrfZY{ zWr%Iq8xaNLp?L2&bFyxS9ROeBqvI9vnH7sQHclrG;b~3(mrwxJ(f`9kSN^xZKiuU1 zzL#ey`u`btdECVr;OL0p{ijwEwWif=DRoOVj0MS7zE zt%a7x`Q3+luq{jf+bDqQ?EeSDfvf*{Hr(X@xtC`Z`v19G`HRi`2iFuCqfk&4`DXhe z>xz8%u0vVmfm`fitJ7);bx~#Yd)F7KlkmYPj9Rin4JW9v=(kDbdsHNOyWdokR8_go za=v!S@`SG)FkWwC(%yJSwX)A{3ZjYaxDOUL#MjlNVSWQN+^S_L)C0m9b+8a`n&DBX zzU*LJ;u{-~c+m)rLS@Z^^WO~unfW)VKakl9>QY}b%h=jCgR@bQeoiV<-<+#75{ zQFd*fil@!}A2@&|6o9q*zx~4fe>mLO|KH0~a{oVNQ|-Oj%P$XF0|+a1P4{C7%QfBR z?;nT?(4Cca>i{|YWt4zc&5aiDp*`*Ce|rUBt^V&>!T#g$a1;OaUY;fC|7V~9yqhz? z(E;AwA+UlHumQ1dO{!a}0c}vVdcflG)lv~?w^y$TEGgg8drL2i&sH6{H2rU(0IZ?^ z2haAOI`sc=_-teUaWBtm^#60$0Ll&g2Uh`_!%)xxdMErLD*?R-uR{yy1H6ROm>0hm zdcf-X_pS)khu}li1lICHO(&_bF0fJUdsGJcN1&-Tu(EcgIR_Jv_I;plm9$@T8RJfY;WWL zb1zQ`|9>W@+RNITUmvs{K&{v{G0G-9R~VqbJrJFMZDF3QPMRtH; zqngFM0MEVJhz7wcbvG(Q<5%KdhhyciiOB!=uNqm7{Eu6Tolq&#>(C380bfElRElQ{{ZKuOd)E=!NAE+`6E)(I+e}(xeNkfw_oy=} zAC0DZqgn`cu4GN}j@0wdCCOJxvHOgYuGDwln{1_Gx!q5al?oEo%6YF999#9btRk?q z$z_y*Er4wlf}fK@uykTqPzpAJ*;X;w>i$Oqie(VZ{#6i1VL%%Qfotx6gTwv8{cnG` zzq$Y2$KzYOY?^r>UhAGFiYA1jZFU_y$x$atU>S7E0oZG`EWd9>6@BUX@d{^I6k-Wu z5=1dS9otpE4sw*?Il01N51j(Zha3I>{XAkXZU;$-w%OJ;cV&#JBmg_=!z{=`oZ#@UZUKIS&+zYyD8nJ@ zya*BoFvqm6?l-`pUe9n+*8DfX#CK4~x9ueUC3Ra!qpT4q?WjFY`v6T#%64`hhEOZU z^iHH%4hzmkMtwCzIuBODrN^Zp+G2fKzfh~Ys#I$^sZ_BuMR`=?QJ^#!&)$fY$?Mwh zj@pfiBarcM1Rg95V^K4#doEAwl52W1;EiQ94eHh#!+L-;7OOO@VN`YNu^X`ZE~_1Nk}H|ob`iY6qkL6S8RB48qCw?P;?d4T4v#aSs|_H zmfk=c!O`3P=|)oV@Ff+VIbA_UQ3I`IRtOKY@o{(+2fW6Dw;SA=&WPkA2tMTL*V{+1Ws;< zyCZSsEv!!DB<%H)IUm%9gw7RD`w~QMD9(oBR8X81C2TA2*Ojwvelap1Yr}2?Je$pY zSDSewCRvY|WO><0)mGmKLN*(HO&fh9PIwUF1kY~Y$PP9%;nUEB%?*65H}K+RT>dbE zDe%xb&3(SQ4eZkLB7}l_ThEr;a05JTHiq1=KkT)3CHd6Wob8{6_P+@UaaaCqVfqhG zo&Ep*UNQd5vyJ`ly*#D%zih@|i3u+^0{+2Us)Z%64}ixU*oRH2P4K54)6eOt+5a>P zx@7bl9=AyMQ+Vq3|FDq%cW}7L|9(FY+ox9KIGFP{K@v@H&I87EyWIkFu!A0pUB20K zpY>C@|EF0n36h{c2Hn~`NV{Q>&qgH3!ZzffF8kqXH$nP3%IQ`_@fI^%a@7>1p zl(wzDv*X4q@x(u9=cmNZ)vjo~MMRv(`u6>lH*I?5xQpTT7FMf>zaq9Cref`WM3nqz zPLf7f_)ks1HqS~=yYoNqvQ@<;KkLf>IUF4974m-$2AlkE_wp=x{%fkJmqS5F;AH_+ z`TE0qEv~Qv31i_Fd{V{9)~J&?4l*Hlgp;rCNH+aheZdc^+5)LbKqHa}iLLRX=x^B) zpyQoc>SbGf7HinnVcfh1KZTt`^uA9_7y-G^G>Smrj?l-hKuWLGVc9f>IK~t^tDMy? zd?*#TmxIzLd{Mp*ZP?Je=rPyDoJj2y?Bi2~gJ8jURM*sfy@*0DtLtlf5tWCus;rk8jP1hF<~6SP!HLZ$K+Pyk zkNW-gRMD8Q_D=anMh^@?LPI2@+olRntZL!d(t#Oqt zxmw$nAj;K^m8`&fSW($|7L4&p_!8$dO3agtKNZFFxOWY@_O8W-tpMJ$AetDVV)XQ7 z+o1d%Xc(nC?sB6!&TznXh1@A`aT-VCAa{-#E#FuqG>RE((u6{10@_YF746!2LTadK zqr;z*)2?XTX~3jysf9%yr82rX&J6ZIz&E~kNs6<8;_&GC;Q7EqtR62(t!ju9D6L9~ zOcUBrFITz0jkqcjjdtS7jWpPbwI)oJe07X7IuDXyY6Q%7cGIy@#@a_g@k1NF#AzH)zWGVj0LJW3u9F?f2_#Hc=(PW31j>nANM!VNC=B5Uv6T`;hEbn znx`?unt&~|TuU~L);mpyUSb1sW*&5v7bkzcwPnovvYtlrKUW`P&T^Mru}y=o%f*4! z^1s7p2Lsps=jr~_jsE{$9=7!RUu}Jd6V45xNk-=AiFbyd@Q>isd(`h|!F6vM(b;0O z$Z))%{=SBtaw#afT?7AL%U0(Ikp7 znnYPnQHFyMO-LM*>nNF`I7%*$wq#d%H{HIC$$Y*@B6`!G;dtI1&w`BReVSq1p9fLW z7caa=QhyeVFSowxLt>c^GJDcFL;(_FCUo?T7PPzw(qI(D5sh%(`|DzaMWNnzkt%PR zKU$F7z^NkP$2Q3t3w*t5+#R%?#wU5nzkI5yvBK{`7pB&Jc?IYL9@ z(<} z2q$bKR1L#4i^5Z~!o!wAc5dRc~u;y**;w?@I4dRBNui%vG*aF7lQ?)?iC;=%xGoQc<0a&nbx#`i8(%z>BKwmf~!T6BH}3B4?hcJRjk~);;qLBk4KxmoySv+C z?;~;Io_8Z|MAeU4KQbyaGOOko-&k`3w1TZel<{X)RcFQ=F}HPYE@`7En9`??Y;&M& zDtM}B`j?|$lE1iD@po?ma@pdbPyu@W);(p}Y!Gzc+P5~FGiB|yTI;csrXj9qxXCRrM( z%Rlyk%;cKvsOrhznLpK}R@Bf3J!Su1>l!t28hm z)hA?^Bgjw%5d8q(EoRb5}A&e#Y#pZsMD=I*_Y_Pzj4EMShT(p3Lh zFJc+0ilYE0wxdg!8lAAoy29TQD+~KykM8t)vdRfT-(0{U*|*G=EgIb&S>B~Vu;L@Y z^^y-{G=yiekgYYA)7q^1l@YC3f$a{vB=jW%8;s=2wQaVm$!A{;v=t_+)1Sdq0un_L zz+~c8r8oFkd4^{(kZ=GuZJYd$oR8B}?+p-x2OpYRPCq{zBfZ5Q<;8Y=QfJ^L7a0vQ zo4OM}Zm)+R1ARzpLVM1<`<7Jh)S?!(E-PpoqN6Fa@DDRR>{zBLNeeA9Ek7oRH0elI zgaT6!n zw>@Z3X!!7A-Tm!(#FNPPh;KO3SFm}Q*A|JJjM7l`W{N?HzX% z>VeRl?GsaLjK@g!UmlH+lHN$Kgt7BgcD&$a%bzu;N3B14baMKoYclzpli!UHkG}#4 zBUZQ0s9fNHlyuCBhPP5-h$eDfFVsI(K&6eo*QH!s+k$GB7wuq7 z_V+6JO&Ff@gzICh2(N*v#3TdqHgXf|YXc>=J+kL??&U`ov5MbP7mCOy>Bv`L=o#`b z_VV-(-fnrv#q=1`OPiHxX`g!@=f>gmF+%oD70yt!OxC47@s6&*m-zzb2?BqFh*X%T z2#p&lbD5YZMv2BZ zM!7S(6^E2CIXA^>&Q%YJO~73BM$4no*^0b_AYpt|o}5w)R($T+C!hguQA~@IOKhtM z?PWHgQx`!42d0fD8VLi=bGXUmAutUZyy6@0`JzN&@~jJ}RyAvPxt_?iWK9b!B$0o7{XTWo_+I?gLK~hM z))^UY+;|c${Pn%m!7|;!(14O;X+58|{6vAY+eJ#Sc=idcu@uRG*+)v|EOo^hsVjA8 zyZ6;|^V9SDV;ApTH^GbkCGuO5Pk+bnoX2yfM$6xf7aC$ey$$Vp7MLCiALe5-h<3Zc zKdp0ztPj&4&p&$4U5yOAbO`7P{%XHG7wNkJ6NT1 zE82qwcW)beR`5=l_>2>adhFM1K(9@HCVGIHMhXoLeya)1e7k(SN!s^|W9=M!t{r>6 zV*1IK#>@ZZ-pj>BooVOlu0pjPa+KkIz3xz7LnWsT!rCr~d4(mPK?r*uCQ`#m!b zK@?WKOelpViJ?Ae@4NU{3q?;py}xKiNu_uA;4h)S+AZlXefN}gNuXNIHl_MOoHx|N z#O%@M?D+_B6fM;tOs$!b>nGxLUI!UiRMuzIH~-{%4#`RhfH*OoHt5q^0kLs>i>bwi z``kmOJ#w}^Zwr~jB_C1^ynjrMl9u(3R+H=}M#C*>XE_s|^ZN1eT~Rkz8Lc>@+gzUW zuZa$H9d;ZU)J8&e)#NUPi=w zj5YJfyHeK8P1wxI;E z7LPG1^G1D;U9E+oZl**_QB7(am!1#==1XJ+gQ(p#aDnL9RsJ1Q6MOLh{(U$^m|;|e zqBh8BOr{h!-{yF)peWR8Xc8B|jnq`Uw9P>*y)CI&O)kXXO6#EQ=O6cs`HhGr>0!vZ z2v`xCHAH}b%%)!u$m;=}npGpOp0@qyNSdG`hJIVjwpJfD3^avyuQ;7yRIB^J)jUF> zazHMb=wqu*7c+}nsxP5^6&OjIJ5h$qlOn&*eO5_1$m(i0`g0cTWqILA_|S#kiQD4| z>WNAT&-A{E2fBk(ENxVkeots z@w6Zgi8n01k8K%p$-LA3Qra2y@m4y|_4CkDnPM@6pm`p53@9>c{=<==$PA1ir}PE& zg&P-Xt(*o+W-1Y&wzUCs<1wFsJ7}y?-h4AfM{hK}uit;XL~e9$Ad;kx7gsGQFj5Rb z=hufE4;Nrx5-lW_$0Xc5d!sSCWER$1eO^#kBM&P)FASB=WU~B#*i;O|YVyjUv^J3c z0x2b}#CBg7NiP;gDgH~*5_;vyMs<7KwwJF%3!*?yPUBpMf(kUgD-UAuAt%_0fovst z2Sry^R6l>9$zrAy!w$E+{6YXjs2e&&dn<){Q9IrGjRU)yGk>_&gT31-)Bl?>>^Z9( zOLYm>*8mD#OvwSC>0;d#6$AhYxo@@Qlxg>=Nv7mZ+l>6f`s!^)!{Al~w7R!sA$$8y zUCzyj-{*Sh>8|~TULb&v)F(vF9VcZ2N*Mk7b{{NbdoL`q+>^Ld{{)AwLw_o-fq2$c zfE0wDyDx7B)<=l|B7OT#A82k@FDNq7Q<0pQM4mqN#?P?Gey%!+mS;iO@ZP(bq^n&3 zn>J;~p$7lD6AyTdi$f(AgF_|bwgw}a*nP8Hw>*T;+1=_nQdI+`RUOCsQs43E+{dtV zDr39*pUz1AM+U})a;16eGov04EO}4!_loUZ7sr{-Nc)A2rT34JUef^-WNM`K>;YMi zt|x)E?{J^2PLieDI_Lu08e%8?Ls1BmT4A=(LcBX#w>o`7jz!P(;7-~&_kE7Lp_e_v z^mod@d7(z}6qEv9ktMR!B)2gl2ZWuBiSdE75G;K9=QOA!uI}3;ddLolwxH)~QDNOMTAK=<>%Syc|HwZS=+Cc^_e3B#~wahlUh{I5_`tO_KrZaY_ zfm-u|r#`0_frn2WpaP80c^*)Zy@&WDz2>v`qgNgyffuwIc)S?DLJVaYKxSiM3ktz4 zQSaS>p!}w*l5*(>+AczKA)UF+ShCfx%!E`AT+#&YTE_=I`bX9z|xIdM_zf4 zTW2H-KKtNWdDjtKHC)?B(Y!h-di>0^E{fYketwVJWF*!pH1DGLSlhJmv$Ssp-SST) z$T|y#&pL~{U_2)UzSa~*#$9~w%@SHmJf;e_>Pl()dw+)1;gG}y1^L1JM$y*>WPy$y zZ$rXW6@pJ~4MsW0L&Rr@=VxA(T9piG z^Zk}e#fu0rkB}@7A5@CICkIsOosiYY^3h+pA;@1@6bnLJI-KNtdxR(a za09gz{omlNqxmC~uO-|ls9(@QV&Lh@mg><^9s9)_XceIDWqmfp?h3{Y0dyCp=9N$} z*4Bf!Vhpe;IYVL;%(jodY~ALA01dDV957d`HSnm^II;3|?%y;Z>kUC5>!M3!!fp)R zAa>Ol-TyrL-!pO%&3M2kVbV~U$P9)Ppn2sdN3+M69p>@*uZdF0=c~_nq42{Wu*LPd zKXIP*h>bv`%wRyJA{fHds=tL;>`R@L5GsJgPeY32fQLb#z!IYgn2ZJHP=;l#7M7i_ z^1*B1;Ya^5=XLXclI^GRNBKHs6j3bVL8%>EH;(?VZ!icCPn{Z##fK6Vv17?WYCy?Pw#gXFP_1SfI0{5q9N~^qvIJ4E z@ij{_*`-o6Ncy_}HpKfJZf0TG>I5`tu!th0i@%}ZO0AF?9mx171Sl|7lQ+a7?xtEa zkCV%Xh~vtdDAUUD+-`f&+HSDd@VZ<>Xtxsh+aP{Lne=w`*6t9+$auSX+0U3cRtNiy z1CK2D)QJdDn0bC3-0C?YNRMTf72$Nt>EU6tdiVLvT2Jr=yH)LIGP|kqSh+0jWfKOM z&*opEgU;~?ikWWIqu-{0q*)zAH*T{Wn7iZ3J;yrsNUICd@!}zh&y>H+&Vs>@R0oYd zj=C&E)~T8usvD%AP0=Mf*>;BLEm#B$);_|jON1XA#Q#NrHz$Cn{Ar#K{5^Mqye|{4 z>`jsXE>L&Yp~e~OiD4$*LtPZ>Tc!m(Uhi{reegiOVUQ=nfeUz-eYbsnJMn6N4{m(|Z(+I}BQa<6QO`S5vydJH z``_3e;uyaQTne}4IvYF^9>KM{v{|~ zT``b(;cdcuD{@=7)|6j>0{_E%Dcjy^Y!oRMHiE`sLvG_PN-cucBrtb@}Ot;%D`g z{%k*IGj~Tne!7>80gOv9h!I~3V-$lWiP%38UIOS*u zBO-{vO5X@NwoDL*T#q}>0#)%9FJopTq5cZE3>{M7HVfOI5`$#hLab|SlG?*xryH$< zI#PRymGCIxu4mq&@2h8?^WhhIZs@g)8&O6p`nd)XN@x%&cFr(k`?wXN^#a-6SOuHi zJ@7j~5*vx_HWGZ0cFiG|P`odh(US76&#V8SvuIZ-Qa0`;czAJszH zZRlW%*To}sWzkxXe)nVMjvg^P=Ze4Vxi44dh!WVCbXB!nZ!>J`voX{+0B-b9v&?n^ z@}wethodM>i7EVkRi;D!$nb=O={P)r8@o5vPYH33+q5k_5@v1q*zGGU7m`%{3n>u8 zS3)L~-PYGRh_8^}J2PH%X_r3p6TmGu-{Ejbn>l8IIqSUn)iOae0iGmz^f z4+zQuYL)RXmP;4|V5#th0~Sblm4b;{ zCicnn1k0{2XV^kE45X0@Qy0(ge^eV4k84QI&WqH=ljo}AplRa3t&HAbh4Mvc6mjj< zZLnZVr3h*w4vm}6RP`W0jA*L`i!LIERw4yfs#YdX47LQkD9Ap1hFMRP(Fjvj2vb^z z$f%8&&m+V~(PxiYNRC#Am5gm_lKrkPz^@F;+;JGM4^~GNr;f};57$f7BI+OIoi$VB z&QdBvK(EBG&?GR>#L>{C%&T`AD8mq-#{GTbKlWWg>HA*{)P#FdGFmdyuVDM0KgyaZ zS+@IX{$XZllKFyV^AJ;07FU=-oFR2knK3)rWW{v^iKc9T^(VX{Oi?lEA z_+0-X&#`^#`%SoHBzsguu7M{mi&;os6hBI?m>_v*6(m_aKt`_8*XiVonlK-kSq7Jk z-dqM3U5SdOsgmLg2Lj|3vxsF|FrEKQYbpem@eA|6U#8S{de4 ziLst0!Y+l=%9T?$9It&0A~t(TRV58W9g(kFV3;V*A_XyHB-s;BsG`BzT&*XgTcy!{ zB2qEl+jbDH^va7biUYY!^*EPkRQQJo?-yBoIrtV#FRFiG5LhqjDyzewj*SO}secRz zL2_Z?X_z!N$Ttv)v3na-P!Ql<6PLgrn*mN~FzO-Mgkndn-%UIVPVsRAuuE)J*AoL< zp*>P9p+IohYcsCvVoAykn^HikESFTB6HQO>*LLhhTEd6gbrsrn36YTZp3)A3-l20jmyN)Jkt-pY7`GjW)}Aw$zm7)V*? z_eqW|mmuCdN=I8}v}&=5oP*VbuYCz4e$utSbCtvZG)JbAjm;$KJ}YG{iW&e#mN7XB!oscZb2-)UltLcm(zTif&Ta#~a6cB!SVeza!KD0ADNSf0c+ zTZ&VBHY#_Fkk%?jkm+xOI$o^PHqR7ix2dUAm7>MT-x2cdCtNTT)o_^g0?u2LicUDw zcGL-1{KIC8MLDoPgCjU-7H(FX#oXi%{<3}GnYwtsJ|(qx!6 zVD%%${AH+@mdn#C#IHpVL=JgTXV>2qtoqjZktX2uIJZ?%a!)kpI--Z6}4iDhrMYgUZDzE3`B;&?% z@)th2!bvwqxtGBv;~M9>3K@nouoo)L9n8V4yN7d;+yziq{U}oUxt(%W$W$_Qpkn2} zR&81yiV2stnxRa~Ze_dT6x?RZ(W1mUtHsjHk>GP5U`W9uL?(E?{msO;CfOWS-J3T2 zS_aFTmc*mo(_XgUTON`>DQP&eMw|<_*+|xAMXH&$%y zD%o4Sy5|{YZo82Q52}2&Y;T&54F2h6ifrY5QSFTDcCfXc&E!DQrMH>iK_N zgsQLoQ=qofM(B^hT1`E_F)>JH{oy`e&p&-tf8Xh7|xLOl#R^S3iC6oa> zNQIyXf*LsN@y`Za9v5`kKekMGCe??g9Hr?ma4CT-EVcOl`v!vV^9opW(DGJ4sQ#{3 z&F#Y_qXs!r>wyN)l65AJl`Lx%%;Fg zD_^C|HZB!!w?u-goiqVuZo%`8gR3|r8r+XyZU(d^s_fjdeh09ABTHs+fPvlOK^Q9! zx@G!J!@(x zc!~ILbL|yZ5QYI8$={T`tCH}w>gq-s@CM8F5}37=fOR0opq>5CGgnA(JH?|1wbTje z7sR@?bZdR%med~%P(sZUqbYDJDa#OOtHYpc>|&kcYX1oHkaK~*S6lmE&xLNeeVz`Y5j8UIeB`6AQ3r+)ASh)GYFN z1^+fmQN#m6rbqua=t*9qlPcIr_j=*Rgpa@l%Ae;Q3zS580o6iC5OeY&C^Gy7@G|@u zg9a~UdA=vxHdZgFJ$P|m#j_297Xuh^ugZio(4z!#9Z?8z9ckEc9YJ^LB#-AmKDnMj z{J&-dZ{~Lp8-UMr#G!m;Pr&Z<1QCw9e!Y~10%o}L9>Hb{^BzlhsF1%txZ!BB-Z7lI zMI%xV>f0mOs0w$87HTBFKO(S-@H+cE6TXWWN!-VB;bmUp@<>E8k8~V4*A}B@8&y=T zn1w>P4?C8Lcm7_kEy;#yY!a$mWU-mHZo7WviCs-u>QCAD<1>0!aB#zZU@Guz<=1G%f2{Pb7)s3~~ zfaNsbEKr4JF6~Z99xsO`MPu;16>#av+SC(Da`}^`@k$*I?B5JZ_A2O zm6T=CP&Sk}Lbcgp47J(*=fHo=ou;yLpo3eBRcNP+?wYROaU6J`1;ndMtt=FMyVXl} zAGnhd)%x!_v%$e0lqL8P3WCaf_cOjXG=dJ!V{;ZBdMTp+n*iVWSAdr%=k!^;p^a)j z$IC98yAF*7fR0Ht_}KM)r>3>(FY9-S$J<9(=|>}FzGc1%p9l@0Uc<33tbrdLv<3o^ zWv3ke4w9A~3(-$qx!kOlas}3D`BQ(ThP1BJETq*iaosgXY$=~qyK&Skl?Guz?Rh7p zcw8up=;GoiZn0(<8s?TRJrJ{jnA7=M-Yv~`oH$q2GLe%H9o&g`+E3E3z6u{#;a)_t z+t8k11+(0;JL1mF|E6Wd@usqTP$sE|&QH zXfbjN)%PO^i6D#SsNercZ1|5MNK(+kzK|;kJRuxf{)G^QA?ed~BwUyUU!U!H{T>Gz zA8v`FuLnW8NbK~}XGgS@99n5FdQUL88>>cZAC5JOPl8;uldZy3%F~fy8?$ z1yB3Tdk{AdZR;&Bbvs%J^YD+eSxQhn@-ROHu-R@FcPTJzsWP_bXs)a9vSr97dv*7+ zS~j+63=bZGnw|Bo=HnqYa)+2Y?Ans2ZEZ945}SXXawNaJq+)v}c=G@t|7)0*zj+4fS6} z0&l`UMgnI<3q>7nAwrNCc^#P6FQw#ai79m2T&e#s65!>pE?_TR?<-B->G@OUzmcU& zEme-}RA)zYP+U05ms(Qu#<%GF0BrCo-ssY(AXdcp8 zWBTQ~+^@XTd|79i)FrH{df)i9Cqq@0{dMzVOj2G|-}$w~9K6Yo7# zm9{0GyrV+RTe%gQ8v4@iysS8%q%X^N8*<7Kp@3c+na++QgP7SUvE-f0QtsCgGNqzz z-wyt|Ihc}7m`P+BT1%lcX~hJv-(cNk2ivGqG8nnOk!30(%a#hx1ho?iGxTv~EQc6w zLZXf4Elv#uuoaXStrQgcI--LzyPIp9|QCiJaS82oXi@_9yO?5~FuT%Aqtn zJ6m)>q7quqKEDFi7Dh1~5ng>sHezS;#V56Q^j3=lb$=!G^uPgywQb8Hwj#f>@Z2`Y zYXgf4GP-R&owCj!PaQ}@C+DX9lb58H7(S)HR>g0hL0v8G7U7-Ocw6&Rd;=J4t%XW1 zYkwM`g@B{~bo5tHA`0G39%pCvE-5PW}>A zS_Fv9_kB`9V(WzO?h#A8svjovXt;tE>=3hkdSc$uTJEaZj7uabs>mAUt+7FI9cl3wlXIXb3(D?b@1R%E#bOvT&9X3?nu0}%8^U~uiOaR zZE~cEN?UQ6fC3Cgm=OfyrJwSEBw_2L2{}LD+f@|WtH8ls6idebblyxY2U&oS-77ok z7~5LyI;TcQdV90|!c{~ty-|e{{mzL6$$m}Ow!&#-0V<2hg>w|L7p3_~=^7qkVYg@0 z{KIoC4YAU5#m40>K*#PSMf8e3RV_heTLUyEK|!vLpC0TTEIpimC~!ly0*z0`YG_au zs0blTuWx~m^z1{Gi`(oOx{kL~orD=DSEG3Xs`G7>1rlxIO*A~4H2<&8hBV*4aT%j- z(*>C;zLji^uyyLUep=bKl-k~9r*YYg0Mh9$zzvD8Wsk`9Q|?`pr@t;55!z1V0m0cA zck72;J2+p^00 zjGtA$94pTqEn`5HebSF7mb<#g^H8}e|Hsi`JRWawZ}bw+QOUn}WHMW&aXtVmY@Xbt znpA^ZzGK2dyT>HgLU4Tzn$PBDnDyrJ(GR)pW1UgquK`&$FYi9`io2$)NL7U z2RN%}Fm?6Kq=Sag3VXvJ16o_)_aco(Njor)nY%rXxOeJHOZ!_?AU0iiWr>nKzW(cC z`h-F^Q@j=5?HKb48<$R{*UBVX1tj8{=a_iS!EY!NoyhbB>wa#Vn|J-7r>*=PYZkRaK#zVPk`oOE%Ey_Vm`c+$P!+P|>J z;6GsxdlLEmWBI!@u4P)Bpwi1KTS&IFREt}KgP-h9$*6cJs;F05gx*1)9E2|#S)gzI zxQH+8~&!$U8(Wj0^hBwhNWCj=gozm~0AWkh#WDga?yS#%40k zp>l<*XUV!ndJ=BUp6$rzrBQ9Pr@p&OSJ|t6K-ue$gViQaP1*V7)%*jV-1+`(#%dpu z{CVE?ydJMS-WmTp^S{QuFI+l&!URN+K?sI!oQ|4c=Xr<*wqj!xwc49q?Yk+iZQ=PP zf{mZSEBG4G@`n`R{PKfaH*#~&3CK54sPzsF)&--mH%tZ7!0WY~{#Kn`@x8(I9r{RzV@c9K!r!bjV?x(lB9Po#D6QliWL$b93GeIELDS zJ5ZdGaI@){gYW}Z1g}8sT;uYCblJHDRslfXfLYK#IrU2o4P4&0m{CZk)4msCq4qG# z_`L@p9HF{_V+KD#GKD|1`wocx9RHx%AQgR!wfc#Dkv|10L_j~I zxhrbAog)=*l%=~M)B8`sv_IP*VX!Ey?I1>CgCNZ0Kre9Swo|U*^o(7~oRjDruGKsd z%3is~k=omv_vz6`uyBY`R$-V?W#>K*!sNlVbAu32$jRp@rjkld{~w?~#jp&1hha07 ze8F0#!+Ank+ocE1-qo#P2p&zjmG5CcwrN3F?Ufjnv2!r@!_Z2Zt3|P|wJ6?2=tZBz#AHT# zqqOFJov9(5Fz0)t)#RdwBrYRiKM$rZcr9_jW}@o1Cpy7qk=Yv!DwvVv(XOy1vRCLB zhRi~zV-Rl`CP{e(7;%HvNO8oVL5e*OP+s_;r&AJT3jpAgikzOYoHWMwQGyJ>=#Y4 zn+9w`Fz024v30W)n1pyD;t!sBF29zxb=R%*HawB(7IW{jcFKBK5Q%s5`AO zrX$)Kb`&cbC@$})RHBi5%=hajZjI6J+a|G6ckaSfL`HNxkqE zSGX264}J?l=9DDUj(bp*sL2YQ1{tlmRoM|L(@w_tDoYU^E1ZXD{g*gqGFqHap2WNQQFM`;DP(hW8ZV27`($vl&oAn=|p8ozQ%DNAen(s@KfjV)JfC4n}<5vWRSKoUH7d-w`Hjftm7lkt|`tSx_O4MU$ zZp3laF1A%P>zL1QB?Xb}lpEB_4whR#Y3_JOGR)ZM9w}D5Bq%R0FL~$ttSK*F`E-3P zS-)6`wEMiX$$OZFGCzTDTH@x$`|77q%+J^A9c%a)+L)1d1UW@iWINX`((S*Sl zhx^Q7K5)m%{{4@rwzF;?JM0lXk3%j_BG6eGt;hLeww!?`Qk7zC5d8+}V@|TR5FVLE z@HuKWFAT6kVau-`iIc94p=dT-cAL*24bkaP?EVL3XZ%s^fAp%<$`!T+^2f z8Au2K$`ychn50s*a-2t37c-5HA5Z0hTJhMPIB~i^Q&mlh1hHxqzR4)QqH(*+AdXXA zuLcZ8g{^MR1o5Q|22&0p^=Rz~?%0qgj2s4v-AHL>3xCHn@V-o)`$ccc2IWs{9gpjGu zn+6rAV{`WG{{$XD`?2Kb#{E(1Pc@vV$7!+f#DYdf4qNFlnkzLfmS_^mMQw66VxC9N z9gh!k>vNm20O~Kgf3wHrBe7U2MI>St(}C}JyvK850XkziV@v!JHsY$Zye%ZV!+GCc zi=u1KDMjexuiF*rf_E#ehmE2@o;s{L8iKY=Bez5Ul5uq0u*|GZhL0UswdaRw%N@bZ zi5UY<6%`s)bKxaz^WiqzS?h73vLnPtnYFi6IP}iqD+sWNRdBP^fZ=PZe~Mqk?15W} zO8|rz9T@+g#eK-U$!++drppMLXD19EDSp#g7on1dD~z&mjux{mReEO9KO*sEO+bx7hl| z754o?-ipVREBO6a;paI*>^}UItT0*Ke3?D;UX_FuJM(flYglvYC z3OA5}tFw&gI*_FQk%G+Mm%s!`EzpVl$=@vO^lFfY^AlPNvB&O=>r_fo{V{2Tz*#uB z%LCf$28n_Eo6OrW9P1k&(r#jPsnLACnKtGuHY$z*OI5Y7y%{9szd1-VPT7BRkPkG- z|GOM6P&WEm!jr(7A4@RE@2s~(+L9P*?;l%GI|}#_?qwX*7lT3SD})#rI(2)GW)nfV zZajaULsWS2%(RTrahRNG>Cb5;b&#ii{1{ww3W;}p1-ubGjy9$??9LBt1J8EbLw!tX zEyQ^cgX-gQB;()fH)W|wUTsMU8|PwVf&E#Ad3rYV!5P(EDR|Dj13?6|Gtf05WiRTd zWG4PRlNck-jS{aa6XXdgil6sBm4I)|p-Hvq>}&s)L4Z(kV+Gz%2kW+y#IwuzM6z0U zecq_ZB>Ntf*R=O;6PiI({4Np6?U1e7EIPj+a(b|Rrl2MZxU0JlXRMdHZ88{iia21{ zZ-sGEw>navD{<{0Qy}IAzFBvO- zOUbV?IOqS#U(MobH2hAtAA-~DT!_jU^vI@t7%s-I&6(n3?(>1G9C&o8t0x`EVx^ty zN4ov+^e~42Jw3`9dU>*UvDKlm=gcHf6^5Fhj%MldFGA#Za+z1=C~z+4(rkN;lWpRKRnkrb zgA9Ellki`SfuItYW!BY2Ur)ts1VKln!c}Mt>-4=2iSSs3Ls>`XhX$&^3jho=2sxaG z&;5*xr<_WyWL&sz(xhvj71N3GF~@sqkwmmVf@4D>-1TRmKX$a>X4{z+0WuqYA~M|+ z2QN7Wu{;z%7u>fCnXxV>lv&gPy}Erm+=#9E)=?Xa>9GFmARSoeE%`SR^d--~1x5VV z<&it!-9%l)aaF;dEy*a6`5gb=q+sJY7bvn$s#Y!E=M2&@yR;b!aq$?^%P~nN#s88l z0Ai$lbM3F227j2;mc={XMX8M9d7;Enq~)TnLGjcuUyB*no!3Bz*&l!A^C**WzK6P= z=@47KEIWhdM!YT;_iU-lKFI720LO-3wv(>YeINMiim1f{tFu-um~g5I^OhqtP6>it z4`K|04Zc>Cb33Z~SH19ULJOX-DDiDcB;1j!=b@?8AtZY)l;? z;T49v*;|sWBTMg4P?iT5HANbTpf83GKOQkHp2Wrro``xfgllpH3Vyu5cnyXch2K?p z8n@jdO;cpC53eHD-Nl<_Qkcs3>*0-5($-0Lfr9ZLun?zuavlwI73Ei0sDNIk@E~5< zHNR7Mr*p&Xf}D zC`4jt`3|RWu_O9f;XTEKAc~tq9$t`+z1)CmU7mCWL?6mqGwgqd}1(ZSgbf|C4={)qS+-Rw<64!QT~bH^Hd>@Tg# zSqr4Sxr2CX5_FH6LtnvfrHXp>GWzSgn#+Lppi?aoY*G|~yn#JIUPXp=^7*d3JT=?6 z%2qaV2enST8r1(_*9NiJ0PIwckSKF8ftaTFv)ms-Iv{MNeI16`-}vh=x$rS7nK$kw zAa3IlpchGvV66?O4(8P6Q)mk2C6ULbY!3_g!u^8K*iAH1tLA8k2Ddi4TNxJTq!4J zf;5N^NL?1n-6UrFq|!My{=DbB9KzY%nRX5Bveaam`s$rw1=%{d9X zXp+k{@d$B&nwTzGOTh3P>u76+%y>N!OZ0B@N{k{H$y?(asv#I@t>GOQbJnS{=W<<5 zx&!1ep(9gd{ZW5lWBO*gHbCv;W$FeN>K&r@c5%*m1jN5#0iT##@oA6M)St^!K5cR% z@UhIc{hAf=)vx~da_DNcC={8&k~@xyF-j%C4gH<%2BuERQC593 z1Sn&dAh3DsTi60M3+A&bd>I~D?J&TS+a$JACL_lkotu(F-mC0if_dHoDt}(_fE)&g zf+4Q5apq+t+Xl9hEST({a7HFU7UFBE4dAN6Ed95tT`0f`b{IAG4)virL{RC%#NaNh zW+0ce7MS(l%M@TmM%7V8c=qZThC*jr&>UKVl&;XuC!@)=IvaWnP;%*>znUzq2AqOJ zL$kY6L@l~cu2%W4&fR^zMJDoH*S5Jl2sX$VI${Du7x<{~vdluj26B~7v35N3f7N6) zy`BTg2#l6w;Bm`ZZZD$31zp_@ul)s8G8}DRQ}~P+T*N?~G*5U2#EK*6rD(P`V}99- zvn$}CVYA^PkS}wTO8Sz{Nua3lIhRERy|BU!$GYJ13t`sC4Tu;}OY$D=PvAV5eSQmI zsaoFM3?F6)=@QIt(&xTrx17BL8R6w5s8 z!8l8H{~`O9@%}^hi3-OUDa67K6}smrrD+&N8}BLOyO^VKkae>uMMXa0djJ{?vY55& zX&J%%|IJ-a?qJ#*N)_2MKmsNj$@YCelEwtWX;VSJGM9UaB1H1@K`Kk| zLBCMx(#>tKdqi%P82B>xdcO-BUj(7yZ(Ox$s8wk zbg%0FAS8I4$SF2|pJ1jAPX@qAL&RsiH{d@So!J;)HLNT{Y7&>kf}hR$o@TWtpaLH& zcQmj2OWjoxRr@+P;V?R*ZhbK_o~2g2W(FPx80 zBES7yQHSnP=l`bFzm1ZN@=Vx0Lk$sg6=7Wu)Jj%Q@JT)Ix}x0jQDdj8S*Z-w(TT6O;oOKKRq=@a(H|0{Kg^fGtA!S$88BvvLPno}iJsv+}a z%Qa5|Y8b+sNWDPkc2J^MaULhy$-RQkr$*A-iv3No)J|GqXe!Q@kf+e2G-~^8^lIm_w(jfQW4ejB@7@WFlL0$sw`{yx_bu*#2@RmM2Ku>~yY7C13R@V{g}>sCMjNABqtU{O={xgkFf z;&Yc@&T4Viv8sbr>b}y_cZcJO>V;7XV%^Aj3hB!S_w*Q~Urrw?%9{>Qvame53;L9t zLjEuj;wLF83St~iF=d@xdH-KfWC*$IGE{zlW}ssW&vOu;(V8cL6FwKE(Rw~4mXNGE zc>VPS@du7lI~*4R<}(R1!D8d{Eb2?t$Ca%Zrmj3&+E(teW5Gu7tNx0K7$E%MR_$)b zK#b-)G5$%OrYoA!Fr7_Vj~6JnpItDoZmCh(v(9=H`$Jh@cf+@bluF6`1)42Dh>36W5d(P8UIJv5-!v@?1O9Q=`&CdZ_u>JwPa?AzGV}89vIcJ zW7_QwV6}Z%ZLirkh^Ebgmq<4>b&)nB2iV*WRy(+Mbagx8u0O62f& zd!RW(@#rqG(zt!JG=&@emNDSR{m-<=h*-|dddgs<=jn=?J!@|2nXqCt&?%?wK|ny zczEpHf&)U6QJoX0!^sXvn;ZrM>B`s%+6HeCQ>6M|dt%2YHqWoWwzLhQ8o%ki#cn~z z8j5`>r=1dSm4o_ln9TYUriF}a$6ypr99xk1U(zSgBOS%EC5J^(7fTDgpSw+A*YdY>GI73v06 z9q7CY6_6Y{H(IH-7VR=ptaem9sb>BA_d2nl207l5-p89@RQ-Aovl%t3Mljo=_VovA zTGO#=9bM}8y8LcZDXI!Jjm~tRM|xyQ+Q>NH8(D^ptzka)JJco$*L7V)V#mPsOIfDQ zBW`QbKR6f%MicPLP*P;Aao&6b#6>f$rTvlt++$W$YOf;;7{6HPDBYB};_h$`(J6R#%!O4if42 zgc~nSSP5?{brh8mO5&*`o93m+@zFEqVnCuQc(Qc3IGHAtopEM~gO<|I0J6nk>gDRK zJ;%va=g?j>c$OyNUe*cK56KI7$Rx*UDeg$H#{Sd-{-7gECQJfuhWxWC2kjVOyQzol zZ%Wvc$*6yNl~>rV{g+1Jt?w*lF+@`4Bm2Fv&FH z*dqq#=VGP5$v64Z-6A4c`RI0OG3LV&A4#fCrJVHB-osU^U}=>x)GB5B>y+?GC0EI^ zjoxrkrN2;;a?mER!PP{JQh zJikd2BqT^ysrs78y2|~3oV{gK9qqQQJ8=l^?(S~Eg9UdF?jGFT-8HzoTkzoS?(Xi+ znfcb*_w2RXy+8KlS5cFK`4-S>jQ;li3>}$1!_?=E@AXdhfe6&q6ezvD*oG5B%=B}u3I>}uxy85hBWUX$3Y9 z?m+4G;JB#h7|ylx_so?~P;1Go>xQt@@R!T=KIyJba+@0CxwP%kLo*T>Q&g`Q>{>R3 zFnp$zK#|F%A2X+ttT32do8Vd!&sdT!bse=yiHN%k>5>TG_%vu0yb{HfYAN|>{0^I= zvL}7BKTbr1h)}4pvRg=qOR4mi+xK*%G*e_oKG@4bG)RV}a5zfhhb0m`FX0zv?rchK z>n%N7nj6AFR@CQZ3g1;UE-mOGW3aXrFxgL-15b-c?8sT@2GGNjNA<%n*H<>?t@=6W zQOhX%lfOXFMr=3T7$nJ&)04HT-7lmMK@Te#AYZrfZKPVoUj1&#Fza0w(m0cX4=C<9 zp=i|XTHRe>Tq<<60wN81>XB#p%DBn^1mh=Z{E8hj>#@-zmbvg;#8Pu%Df!48lF{z2 zG=I>vD6*;@C3no5HAPg$QR9b7Tj-(7WxkY;Wg17&VxC-9DQf*Tf!Xh%O8)Vc>y!=` z57Yfs?MES#C|VK{KR)U_4`F*7bt(Fo?KihA>(1n|PY;KSMz&RQcV*AU9sHS>uJw)K z9Qn`Y&48D#7>A^}PoP6e`K5XNLb^n#9gWrKY0G7@DF^zJmA(fT6>{vSvD>iPnV#JS znw+q7PO_M7pn=8;Ae+Z`I ztc|l8TnC>BqTaDPZ7B(rRkxfKHkwv(GvF|rrOH+hPL|3jO?z)UzjEPIQ05gHD2T50 zP|{(c-h@>v23*Fb`C$@vq6WElFxQZejB6adPXBBu&GfKfOY47pW_Yaa`{md!ZC2JN z#k(w#A~NsFx|tRsKS6Xi^W8`Tu8!=brRYy-`-t5Lin3=5ova2%c1BsRQjOeq*|&+r zA?goJg^wcHVO;YxT#c~$vTqzJL6(6g>1CM7nE7#-*~l#wh(>6?Cya%=XJrn#FQ_10 zrG(qumu%a!38uB?~qIG>r|k zVZiOxKT>ml+t0;Gr<4IlVQqRH*7(hm^sosLza`%*%+B}G9vq#Yze$Ivl0;7> z4DRyqtF{%RkBr}lOx#rN($>SijxGGra0RoJw(y@##(YD_@oTz%d~XGdw#|PMmT|E6NO^CZUzB8kq5ab#2!=9&Q;=iaHT&<_m-ez^<^G zO|C9VO=Ssr$ZmRmIsfg^3UdF9HfIWca{Ev|$y0e$RkHhGu4R;OV)WrZsAw0E!bghU z%+-{$?H}Cc)d3EV$+mVkE@Rp9%;HiQhhj==(Q=rxO)~ZKQM;Gkdw#d9rL`kGS*vXI z}XvSMtX@Y!tNz8(Hpik*%JX zEeugu<@9tmk_5C5C=-aD0d2R)1`9)3eg_6q+%AM5sxJiryo}jIe~mhu{k$vc$CCcF zXQmUci+op`R*ZLnag!9geawf1*sh0IJkA6=`jH`l==(|j&Z+Ezx^3)Lv-otC1IW68 ztVWNOF6wzGl=k|Bkgl;h1Z*Lszs6w+!h@}{=nkbyTI)rL4JQL!(LUC5YrU9RDrYY! zn=}K=(T=4dRCjI*4hvU00N2E4)zpe=L_{4M_nYj+4pevWY9M?YZci{RuUasCn=ia2 z^_;Y2j=Zge?3IIGY#TS{BxuOR-W)F>(8H< zrG0l0Q}9mTKH->#JlX&G)Ng1qOLvH;eo7>8^Hf%MU71Z=Kk}ri`Q+m22{m&~&pCUE zMHX%feRd}Y7Y0C@Kc)eY07Q;|X&Dw>I;r&uYnRy6Jb_$I84hGTw>l6jyDXsKQMg^u zgvtJLuYzvkO_44hQmLDMLlPQVA_XtAX2)1x)!(!UeG%y;$YiHl`nrg&LxFXF)gW*! zeMwjnBwc1L)QIGzTAp2a3ZqBgy>b^5b8)DJz4(wbF#x>Tjgz2@Rq+LlrH3Ow&pS=c zD`jR=_I^RLy^nJjXVG@AuUz|3d{<-pFvFz)2-&KXXxKp$g@Rc z+Hp_b&Tt4X=qvXU4AN3x3Rt$4)FH)Y<{#9aIi=KlNX1GX!wdAnjs4x|4wL>q@!`0$ z=Y$K0eG4#*;~2)4jnKk(g*S&!QGlKRS|m58Tny7MB0fE-o1%z1!L;TQt6zD4_fXmf z(&zJil1)~+j<`*{GGDlUOxHe*`k4RhYFB80?%5uCKLVNrJk3C8D0~KSA3TYrsk0Lk zu`&a{ru=mPBs9F_GV!!lLlf=-9RR(hbG>;&g#9n3I+R0aByR%}A7YDRA0DxyoMDy53TmwxFgB#_6oxyO97pfe=^|}44RfvD6I~gIUmYJczx%&8#s06<9`us`QhU-vP?SpxjNC4BIl z;iU#{3ktV!2~x?Mw`5*`&M)Jvz<6C{)o|M$x6B!!7Iw7ZfeRYb7@VnBoHh~YSp018hY#r zmU1A^kUFY$><99j!KTV;aSM76Q_XIYrfL8K%Gb==EBvzCQeOyUMLerG{|j0V+ZmQa zY(A<&I#+llMC#j|gCVjH;K+`pcU7W_5G&Rw5rh%m;T|%~ud)8?N)KZ-{-KdX=r8dB zc-$i>rUrh|t4lf_i{!FK3K1+WMt+JXS#Y)0X4;Mke-EFQ_13?3KackL z13MQ5I>*jLn5V1jrK}_7n6tvrZ5PH*DiG(%WP%hhTBA0H69=83P)1(is0Yjk7Vv&`AqN zeTp>7;N*1)JA3G0jdExxUoHeonrVX|-E|^{nG|0!#K?=*W>38)1EPZ1<%DkCHTtX3 ztr|SHGI}?a843?VSqLo|qf6(X7NcAK}!^ILmIWk*dO7r zKGbR8f~sj^xWcL|9l)JOB3eP!Z<2zl+^v*is#&%K6ddjdl{Rn8*xXJ0GgcA$7RWAW zzm#`zE;fP=Q@X22=eE(Io|L$Lz|LuI;_CO-u#b%wD8PYnUPE@ptEMJhdMQWms&%}S zm_>goaAR~qOs!Ytw(VgpNv*O`vj*Tp$?L@7$Y&jaCaOumQB}v|$m>LjlM1>~3As`F z-T?OZ2|%b2<#j~rvZ*F>C?F!$%m7$&-_v_sHys8*0->O*&p-oHav=lNWDuYkH_-)M z4dM%tbO@0Mh}zdk0ox!w5#OC5ww)9r4C|OgX2^=pH)1xBZ|~`g#E7hWNH*tl7#2Ui zs=Mj0LeD^e)87o&YHuKQuM+q+yLfgIgQCW29YYM!3&RgprsKtGb<+B-^^r*tbEZXq z8G?_+CE_F|Cu+u9SA9PCw%5ofl&bFoI5zf-n?N919xJvtsJr=m zJ${P=*1%dYUJo?O}0Up>Pdl60ayvT8q{p|{A7>!g@cLZATLjq3ycRWtb%}Q;Z z>%Hk7udmUyurnH&n*N?c_mFR#LkvKUMQ<fjjJfXN}N%WRTjU&kt8X2 zu;Yi!pwy_0bO$uvJ=2uS0rx{sU#Mcf`?DXjl%~KBVShW!Q}<7i+0X^@-}Ne}Ji`Ju zwX2#XNY#obeV^b0{SUcIU;At+Ds?;XqSy?u@@;yJQ3#qJ7Z)3oC1o7Gd(P`uxjt)j za#Pr3>RfTPgTwMIxW6}_Z(wl*i_#KrhKqVE8s)LjVrFN{43zPoA%qp7@TO)Uyil&| zWIF=2Ktx1)T28m=riyLOn&2QtsfC~ThUPOCh>JhzJ@~CV(H7f6ajRDM7e>e{(6>if zEXig6`0p!9en=;eqSIk+j|PrHc!9tcCBCK6r8c)U>&=+U#{9PGS+Z4UDcqYh8usk) zkUF@2Or=r)bxyUvF|GU2x!kdTgT^z4SnC9#Doy>Yx^(V5wY2ebTT@2eu1)n~*T%6% zS!!76Y|h+g-pxW5=xnJxZsjh$Q!~#vH`!@RC@_7UAgQ7g#kMB#|24QfXLiFCHvFu2 z`IS$KoOLm$3VXbxt|F{hz1}MVe`nR4W}VOZ41Cr*F7p#MfftV6N%Hf#{$$Lb9{4(P6{Jalu>~}(L=uhhSnp7y* zUc*GkZN_ST8hP+9;etu=2MA*KSBW{jZP8I|72^ewb!uC97T?a1)((oYE%Sd%t4PPMK)xsAflmx8BlDr4g&YCC3S*p?BaN0AHX`64 zKIZ)QZy{G&**qmjdd}HYS2&Go%b%VBwbAF9*rSh~M_!9&4~Oc#-mD>eztCAw7EN*p zV0(aj?x9hJ!;0V(K(k`yV_e3UB-)*ZZk>~-@{YIC$0^V$?iBgQBS{c;zVd-%-PyEW zn0Q@P4F0%DF42Y-)+4Zem2aXgNl|cves6tbnCP-81L?+Qw3PT#)xPrfXTLmEUUn$Z zH6vtOQy5~2kq0+2lq~_hj&RW3X-B}g$EL%*$k4=_z@q-O#?hJU{*F0iLM|2ANNz_) zT>R3`3^pK0N$f)!V*-I9_p4mZ&mF#yKP!HX^}WX>a<$GkzYtpO&jIN@pfuz@6tHHP z4oc02w|?x(5+BkFQwSzy{0BJxo^30%5I5&Un+%JfH|J!N0iJE9v9^P9k}?oYnt|)f z7vn?lLgS%o+R$>UUVGmcgSdbyrsS(FVbZgYPPCYii@05Cw{7fE@jrPi-@b*&tb z`XCu+-{plg?%#Nl>o0E&Tq7qTvFo%ENy6SZd&LqALvo|?!rq$f+3fYxFP>>z1n z#NBF;Isx9d$L;N|Telh0kLu5f!^Q@QkDANg*re97wR*8T85a1*FXQ5qX{}3f-mF^= zK-}HaTa*&xo8@RXS&jqtJwu+u^(Y9X3C{dr7^N>N2;~6*2p95C1Ccr?uIn`L6A<~x zYA=ZJs4tcPMY>CTqW-;OUo;Tv3tC0GwRL)dD(R7cOXqB6H$o4=F`BZCXhgv9wIfSvs?EnxLQ80oc_&`T2=sXP(+u@Hz(%8$mGp z4ojYBgC_*Fwe5dawvqaJsK3Q_h~Beg63FxAxC`C`%dC6g_dr&CJHVz=dGQ5FeFwZCqW=Eui>Q6?-xW~R7yJQt4uE684Dfjyi$KIj zvJ4jcu_g-wRzaZq8a;ozq3wuX#{mGY9spduFL3pMzpDrRyL!RDtB(~1!~%$bR|8-G zEWoP)&u7sO5a8IrtBD{sbO3P%-vDuWu!4V=KJ<6#aY1c7=l4lNo>w0i1g?u=Vlrcc z<>VT^_d?!sk|0Ecg8aUo$mG6nbjzp#o>B6c{SO{%zUhxx>PiOJ}kq2Tyi{h z$-R9#)z1S@CVYkk5XM-fLiId`M}i&MmOTA&VHn3`(YlXFx!6Kpq7ApsYVO!+^XUMN zPEx9&`h2jRC}qI_xfOA{?Bfd1iTv88rIv1@=&FV`pZbV!6K94tCSb9Hm$8UrGEV3>>6xToU@9J^&^#Fn#0s%py zn2EbpM36*3BswAlTYw>R-h0B-@liXUN5-EM5rlR593hek0Y)wm31FZJ!ZHZ~k<;CK zSE&>s^5#|QA?f%v342t|vW2K}<%&W^o^xw&(@A0ZOCJjC7vpe51Z|uP`N=n4!C)m8 z_xPgmnRN0O)j+_D<>`9c!aVxR!z%1-`<2CU6M{)(i)zB-0G6qkQ>OYp7fhvShjg&< zwoaw7!<0WhkLE4=JrDiH9x&Y%&zQI8W|@|gZf6hEft*fbG4vAnmX3S9fz{$R3jUi` zje#AmVAVyH9$c{USGIN)4%^<0u0Q;j_j@OL*-CS|ulRP<;VXcp3HU1P3$DMDlEkO*a?)E6giiO#Q3yH(0gEng;p3`Q0gN>eW@oxKYO4W=GqZfzJ3BkG`p$G=pnaKbhBMt--~ilM z`hnv9+OCT7aTlX?)`a2Rtwx=_^|!pM)w3hR>iSJ}d}Gfx6ROYlqec}~b*&2hwZ-*I zvIVtUc;?sh>aTL+5(EyV?An}^Ngp&(M;jV>D?GPDgkoAoPE_?q`kHV3x~nbyi%YaX>%N@Fm@`8W`e!B}bvgd`W}vHx&iJWRsNAHzM?oqI?H4 zwUe*%3uSDFw}MQdz?o;EVLrrFnBHp5*&lK1Asth-9trNbJy#7>%q=4ant{mG#xrtk zNd@uV{|{(T@*mCtN*JaTjhB49`3MdKM zy!S#7-~O~DkMVOv>gFxeV6u+87GRjN?S>SUb~A(V{k6PVe+21xob13f{anreIt@%Z z)&5J}YdXe_6pJnRH*@gI9U1;X8RdjE%qY ze3>s1-dvvg!+Q%(mMqjcheM;#RK6%*P?1t86kJR^`iAA>d2A3FRC+Y|{RdbZ|w}zs>9DrYp(D^`nn+&yA0npx-1(xCQlg$gJ z1+#p7M9@{;``X`jZ`-cSJwcT&{Kl4oM zv96hBXgm>KRrA&^ufLC4#}-^-WJo^|R8;0?y|X{nROZ)wUW)@}KmH@n?W+JS>gcp<=N1au019nppxI+0p zmO81A(5A+av(pZV$)G?T_We{K4!BIoDKqeKl^;_<(jj`rl}SVMRTSH6FTM`l(2{f3r{(um3X( zMQ}Oz1~j{!FiG;HE3#z|@;bzdaqWoxTj{(lGwK2=ov!Z=v?sSdgo$$*G?<8S?1utf z<1y%Yvs3my*PGAn(RFtm7QUc2p9H|@f6b-izCY~VKW(yn&qCWqDN>}S(>5L8@J{d$ifjxO^iwrYe4|)ert)Ix} zLGap>tp&MHakzdEBFuT+_NIbRc_`b1#FfuP%6x z>{*b566D1#gAyG@B|nsEX_T%uq*e0u?0Z&3J9b+H74BN3vGa~}P!8U+Lxu_oap@T7 zv?xi(uNhmkQ+d3>^y*UwrT5o^208Ewhh)myRS&xGO@Abfq`7-m(F{^=8V)^CBN}=J z_y_VHS^i|Ii?ck}-{ysV zMQR-+xl}irgYl85mm6dHK_J3k{K-ofOk;1?*Hm}~66TYK%WQo&;A=X6cY8Qk&P%ht z8k_3_OYoF<+5vUy<+0EP#1a1a;I_B&Y6#U0;~a>S`nUoMjSbg^c~BG6j7z%8w~sV9 zQ6oejzZRgmC7lx*PWVj)v+}e7jT>C4-}%W7ge4DWW#=IarZkx=nn12Wv6u}SPr(f3 ztj7sKvL8Dn1MT#eP|x+^H^qoIrXG88n2jwJ&_Xd1(yJYW9i#`RTGCN^C_O|Qu#=pI zwek7w2#~A=;enEZQ{mdxauj?U*94%RE0ojBfazMkJj3-XYBXxg1zik8FG*r>E(^q< zX=>>Fb!4&epK}BeBmm`8Op!Ho^nvsc0Zfw7+2(w%oBn{jlIzsp`Oh!OdVD{629J?` z8atPP>&jp3gioBi@nWXnjP%yyCxvuaCx@)7fRkXVO9Tq3vgyzT792d0ZuZU!BA7R! zrz}A;ZHBg{#*wmhiibo{EwLOYjngO-5adzZy4@JMz8Erp+oTy#PFcHC@4Zv1%8j}>c>~*7G+>EI{*Xscikc8lZJgFg7*Q-Ov?%&4L4GR~L8=kn!~8bX za%7+k!-xX@Y#WVyXyf(p%gKx5#h{u0t?nbF*w5_aR(RjLng88lo;wC@`i|HFj@Y!Dp zsa*I$G%9V6mZaw?8?d_w*u|V}oY&)L5kPQ38@sH2wj=JzGecgyO)dvZp$Jny%ev}y*_vYu7wv)vE(fAWx;^w5jt%0o198jVTkY9-O7d zu#e%^SlJkOa+s~HiMYT%jMJx26cc8mM`eYDb9U;Yy@1{xR;k0xY?0YAKu;MrjoCE& z;9)LCS)7W^V|^;o<>4|Zlo{D5s_yIiEfeOw6iO9rBvH5=3@T_Vuw(bsfLa?! zLmzS}Svo7YvFh!Yq?;5ep5@g@K5{W=m(sP+-PgEvlj#2*2q_UXRLqV`G1^1NCQWbo z*a;}v+3|7p>Mx0ftjpT&cWoFk0s4bdfV`Ue8)@S|&Ts^4`URAR!K8*Ci+B=?uTv{NMxcF76Jo;`?s? zB#f&BcE!%xZOU}}C&U$u!hJ$S18jg~=sOs;I8i=090AeR8||x>!S>zHhdlL!Vz5QKmnvqi-1p z=%53A1fM7g^%AUnkTZY!-!eJ~grGQ%AdBi*C`HD;GKz9$8-PS!w#4p$NheG}oyuI} zM%XGK*ESS?Z-cPa3qN#s)Ad`r(Zc}oZqW+`E&5{|rw7AqXqlDq4u?V;ZI;zxEx$1! z=$TY!236YkLVmJ$mV;Vlm3(C1$cL&<+KH3i94BKXoGB7#;|wNNow;b(u#+ydOdttq zSu4t;M6PBrYK7RpU?*jLhkhQUAeL_Xs3aMGey=3P(&YW)V0z>-j(j0YYF(ry8w1rl z-a54Z%%3NM(c!Qxu8+7drk(yoG2(!#(IRV|sWU^zEGaC=d5$rj&hTh@hZm*%cDYV! z^g+%`D-0NekLgbUOlf36Pa;rx&WUUY<~p{>UcXr{8?44w(UUsq)ZiBR@V}92<9}v^ zzF90Lp(dht)h*d%T+`L=1g|b9u&(LeuAWt5PP;#h!7YK=_i?uInZz$*;MY?L=Fqqi zI#fT5puP8{q1F}Z!%8oMXz5{m>BUG5nsHhKRr0nIj0pqoe#LN;Lga{pA0bcWE2B&q zN+L|{I-vRjctD-mkk!xxqgl)aqwBz84Re`@2EPH1$y>#I@f1SwZV#e-@j$pi?Gq(I z?WKoR0Ne~J;k-*1nKm-P1i4}OLOU?he+GzT6r^X~dI{dg?6cC%YN@m}>u^q_RPr)N z*85OVk_*qgu~}si5X>6*%^clQu+_)Fzki8EBQn=XDC>~-Er5cT@Bm1F#22*!fajG% z0c4eAWb0=uRDhX21s}mLv;_chW2o-wK(_8_gX@lvd?{1%3xU0i;g)q{jC4<-KT-+g zXoh9Obo}s8eoGsE*oqlLkkqrc1wJyzn+_3r5Y`uU&}e`7A)XJ{L4SzN=k~+5W+S}gO0XxHEAvY4bG9%kpSuJYUdj5E!M}C z_hqN;PBw7MY+otZa)yNrknFp0R!e1a7a5GMwElW`ANXB0z7&o6_PO}^ZPs9W3Fq~* zxIYhVC*s^o5ywVhhY5WR2$NP3?1yxAgJy%4W|?f>dE^zno4%Hltu0@tDXt|lWeVnK zR)VPuzI;!dCgkTp-OV?C=XZKm@D3n+kuX#vVfqkhuL&|B>AaMTzWWI3?{`*Q_D4ca z!AC-7L<eE(emLIKm@sdf$WRe**-)<3~_ZWgB`Le7vXJ}oUXYqYJzwb zt71O->~tN83}huo_D{U%P+5~`Ayj)lg>3nDX7hU+M*B%~q`SFB#cHyy?zU7pg@La4q?IK9 z>67YiGUxczq^AdwE&_g?%t4ViMg2Q7#mX1FF>>KI!Lj|em$Xp^u~g0`$fkH|2CG8P zy47eIQOnFn#;*k~vhD@d2x2XZ1LuF-F5k}TM3;uC^vNVK z^#!<^9we>OyYkk*?z7cuQnaljBhaxc@fLlrJXoyMwu~3oUQe&kO2|Un3vpD;av0!r zV6>^r#5r+v*G%JynAc8;-Eg1S9^=IzyrNm@PC5=^ZR??#zZ_PfR^*xrO=8be2H0cb z)Ov6dw-5@lPunQhE0<>l8D7_eO{)?T<4NKwJqIv zS~KRK5zp@PaZ8pOsJ8}!c(hk`{HO_Ty+Xa7mm0x!&t1m3%400r%m@czKaLqD~8L0BEQGH zmRE?rb>fSx6Hr%I8f2Gh#W{Fa*PYC{kdR_MBV7yE9C86D#UV{$8UloyCM$;S0VOJL zA`B&Aj;u^gxGD0zP{Po00I}Q#`D`*+1~0@8lmR>Y-rn-?cd!QDAmC@rf};s866VkV z{4TG@syR)8)CvAy6OZ?-nVE1LJ!VxA@kBic-Bu6FR^EX6I32oPs|`Lqs0s?LH^GVn zLOu4G2oO6GBuY>_gC$BN%7t}BKQ89mWQ2|hGgf0Uk3KAARsyjq+in-NB{P9itjCG# zy_rJ4IlnWDe~D!ddY7a z+zum6h!+_=eY zhljCVbWSX~GHG=hWW?G5fvTB)vETp$cE2gDxgpJ7YrQ|QFJ6xR8==*s&5^xwFRVbY z6<-U*R0+=1VC2T^WlU!zB6*dV52J!K54`cKNyy zjB$*wEK@Ag1(2Gm%U{>(s-$;`S>&5=sI-h}^`+X_rZ|8vtu#&8FBj|2kEj(v466z_ zcrmU?f03Na5urcyMQIFVODsW;5DvpBa}3j5Mh3N*KhY{rc7du;r`H87o6pgt*QF~_ z9aSPNQ7w0DTye_^rBbDx3Y@o?r>U807RIXjWi-^~%+1sZX~mcAlRhaxj}cv2`{|=e zCn@u!&iT1RO<`=R1qI)#RtmF5+fDzH*0MD0W#T(kXE=)V(x}9sU2fot2yo1D#0P13 z+X3vOM2!gl{$YaRYY*55j0}^3jF9)!!0i{nIQi}aMy&y(m3@jJdJL}@fU%$suv9Rn z=4w5gW2+8T;oElJ3f~J>s|nImuqC(}Y`uDA6g>yUik=z`9a8WX?4P$33xaCJWngz1 zUM{{_y&IgqS@?pJzFFnIS^Jn!(kW)j1hbhSMPSk?C!kFDdS8j_pi&v~$p{7UeOuxE zTj$LZK^BcS@wZxQT)LfnPan`1=`$-A%gnYO>%I6YqApmYFF>Lp@iI?9f3j_v0k7uz zn(YX;#nydHUlo_UgbvE&^so+6gx~=s4WH_+%iQ?3<9iKUuVH)pGMyf8x2x}ULA>vE zyaD=vE5wg3zztx}+GQN=XVwp-pIK*s3^gJ+u4?(#XfE4htI9=L2T%Tr+yzA4!KQc1 zJ!HnD!ELwtpWWNOy&xe{I&!6IFevDt1m5=N zD|dEnqSj7n-0ppvN9?3XA1f2O2!9oQWw}U~z5T_w&>@k&>1ijf2|UjF%=Y*c$X@ng zub+N9h#39|dnc4zsNY9;futAc3n(AtDzUgiqeRLGZRS4P>Wk@WcXN4LXzsQB=tbj} z<0it0^KwCofc6Mk_a?%r?V5&u?gv4q#7Pow0~2iXbym^_tUxr-idgtNOM$F|YuLg2 z8W5~iuGRw%qMEs_#$LToR_cfE#v@twM*SGKHWG=pbe2IXCLnrz-C+w)L)}v*7A)aE z@Ef&xiH`<#^_I0aBxJzgg06#LLm34uK)#{CK^L%J#Nc zTg&lz+Z|!6ZdY{Bf>M(FVVQIaiDg=5iBnr=hoU)p$Kpvpt!Yte7H|@?)N>#;aXF_K z%h=6GwHey;v2f!jY0@2Z1Bs}Tcy*uq(1F#TOG=^LQc>9KsY z2$I&~*!BYu^zmX%Ekln@2NTUg$Ufz5X%Zc(b--YfOb{Th#h$WL5Ku$BG^(>YYFUGM z+)vgE`vU2=jy6IVy8a_z{W~MZxT;&rCi3=&aATUHlwycJ3+Fv++G2DwYykBS99XS@ z{$|gZ1?r&Fd_+E8LDlx z=Z20eHLJY+5DmGTkchdP{{#KF(!$4enVGHqFzm{7lGS*@&>k%H^fi0&r`7r8*EErMX1Kj*^476sFG~c^f3m>R`T|D8d6`Z(s&$(Q;i|03o+CsOzZ4d0zoC4-&K(9`R~ zaNEogOS`2n)XtX%n95XHbRKXVE)MkH!WNnCC*3RJ$-PA-fuK3%W(p}CO=ia!Nxrx> zbp!Tb?)#!dPZN&2fkv5c#YV|M)Xr@qwcT%%VIGB|_ARE~gh0r$HNEiZw)T-}JvxOi zmeS~}*G?p+p+l)#R6Y}-uR|E|5gXA0<+qed%bnlsh66n((#V4IUC;cI!O1V#cTWv_ zpJY5MtvKt`qZ4N$=kN6A-M0n(4jc+%6cYGy7c*L1Bk9-f9HUB>tSy#)MO&R&=*(td zD`PGda5JAFbENm(97^D5(yiQb|HTa$Oy7T+KU1<+RcMtl&ki+<3E5eODTwL2s7%S^ zZ0;Oz7O$Oohj`*R@j4i&*KaFGO9<kEAVb-zi<8l zGs02c3RQDw^e*423aY$GNhM~x9rEyoy~)LmyDh=cr0zFc-zQ~7(tc%K6O-ZoQ7r*v6(o=fUn(Q zi2hs}-G2z1Z)WdVc4wS`UO1ba#h;L!%wkUEb-Wz?cBi4mKu=ns*g8{>Pk$WBU$j|h z6R*EUmF+Tkqs5c5nf?~t#`kP~EXxdbyj<`6Qp$?H?_)hckTZ5Dr9=`Hc|DfHxgB}b z5i1TO)(Wlk&e}&^5>#b9o2KBu0lA!666B4r2K&|9kaBc4347OEf#IdaXTutiLD`xY z3l1^`d>3J%VI!>!r4jY9e8elX$~Dx%jxC$BPx`Fin}DHdj@k~l$ohEB*~xMSr6tCd z{(4(;8n0>E+}XX9Olx%E`z`V>u?6;(bLZFgMZ-_q=GEa(m(~uO!!yxh7yhGWkARPy z_q_G<$JGGLl8z`$rduj!-G*$h%DdkAy({$g`}=9cJ`#o^5#f*TejiP5L7xLYX^JPV zYL&?mgz2c6vZncI;y>n_sA|oW8pl-w<#3yJT`bz@IZQ*n&9);}IZ1J$!Ok>7)Y;r}4gqc<-6^1!5P!4~0Fw0{hA+N`PY71eU;LD4R%;dt z`>Ii z1@(Fum}{6IF%G<;uE}KLjY{nh%hiF+-Q;{V~Ixh7D`mODjy$bRFR+`YNUz z8E&pST`u!Y{$)k9wk8Npm`aKltBkT!N6585c29ilmpI5y4C})RZM{XiUPhd#o(pP} zpp8Pm1)J&rm5qjwv@tZvh1n)2dsT_=qdd_eI&XXt!8?hrR9LPcc!KC)7mO$smog1w zTn?$gHhWZ|Bz2vM3NmWgL*5KvwA`dVGLU1E6xu=qJBaDfq|TrG99zRoK0-%eG2D zlFvfF?}!h%C@9V69=$b%el-KPj)w@Py2_V#!|U~`JQ}7Vgo(2+<;q=@YIJR^xg@4&E?iDf=U{+}g7-Y!yOITfxX1_OaU_|FS2qiQvab=NzRVV_G!BJ- zK8_VV$=B`p8DrZYl_%f8bKkkm#0qyM;Js8RfY=|M{u0(5BCt!Q1W4S)iexiFo*^&$ zcN4_XBc3656DVjrKqsU&y+VVd2>tppc>Yw3!~b7$o~Kk}oi1PFDmIeD&?=tu&>3Zs zQ9Bk4zCeRjfq_%HL`U=ky+33-3VV1M^P}1z5<*|!O#v}Q(|xM?+PSd0G_VBiH4s&( zB@pQ+qDejhEg`2tMlV;?11}-}+u>{G{j0;rzJJxRRn&Ec`KQAdWK#H7hwmQ-?C_NV zhyQf=gChTQ_-v^E>hKHFmVh0;H0nPcem7g(f9vr5|9^D&m{{sCKmGQ9z_oO%Dw1u1 zN^Q)@{_f67tLkd9dgse)FTp0qS_BXoj#xNORp{Y5ApSPu&53`GLI?wns9^wb&iW|? zs;qPSeY=RIvkLig?9Sxn-HMW^d0X0vQsb{IA3#|yRLUNCQttQYi$CR0X(?2i$~)K7 z7Sm0XMh_1c&t8^XZEeS?JbEZ0-1t^iwx9i-#Es0&tY4`ZIBx@aaW`h>xZkHB=B)?k zze*q;n11nzDp9IyoT3knn0cImYT*1vtkjbQqhV&+;0uM)+qhNnikq*+(An{mV^$k2 zO%g|mV>hb3Egmvqqe&7QFovR|&IGwKG@PjhcMR1T{8;>nJv|=%&Ymh`*yt=KtU&Q| zhrh0WE3bK1Zu>1wsmVkB2MckwgGu0aH@-&>xzMo3_|prJwu`DuyINmSL|VvZ$nK?{ zBtk5o^Ne?FIAG?2R6pziqYqxmr$Nqd@XF~)XNsUUx&YoMpFq>b)~deq$VJWkEown?1LjX#+I z914FlPY=w1#2858P~=usxi^pOpoO&!_n0yB45gA{uKJ}Knl2;vqkb%Z;`0g?2|+}= z7C!DY(_CmS(fEX6e`AelKWru|6?=1R+U%L6taCGo;v4372B!qLU6Y4)p5)Dy z-){?hE^Ft*a-%n3Nq%={8a(fFY2i0r9J>|@2e*aCQDOtGCR4wDI^XIH<%3_FfQNlx z8z9VmTWmTWqY|?1chB^>Fx4VGdWx!av-`{&!-7hhv2Lz)XI>E7oi>}o&D7qxe%i^K z{Y}JIneROBn(&H5c3rc#W5#bA3NsJO)#TH%q*&Z%o!V5Wv^rlYA>}Kz(-4K?4sy=y-6J5zpSZi*ilfQ1$Jn6yr@af!^`Sz*NyiVw->*#(M;N8Y~gt)@E zvi_m*e>i)~pg6mATN`%>?(XjHuE9M(a0%{i!QDN$2X6@O9^5^6aCfJ_&ik&l_FikB zb86T3qq`ocE^1akRQDYBxW<^>FA$B*R}dD&gXhur4YjHMEm?$JPmNu|&tE~uTBKP0 zBH%Xt$~xr$ANZCBB>62v*j7m3SSTJEJ-pZ>1YQF>7os&WG>l=1=QYe_DI7CU84NjzE; zr>kwd0;CL>=iI??Q)Lpt4(`1vXL)wMX?(`Rz#S~oAF5@FVUv$@Ew+i=tI+%j*ag}Q zu@Iy?AN8r8wWH>e7|Ya4v1&3i9?O5*HXzWXuunN zS=ye^RTy4i;+G!P{=CbeEH(@gatXFEi{lVy8P*#Js&5KPp*m5U#vyUmq30Jd1(c9H=cPqEXT?!T!rYVfnC+IctBri78+@E}g2) zY{K({e|zd`-I7cs==e;4uf*rrVpmMpQ@tlPHljA`n`Dez=r_Iwx=4GW4Bt9XyKhAM zE``zJ5~d+K6E?DwF)$jl94uhLzu~tG9H)zy%3J>p*0VzJrZs0|l;3prr9)&S<%}ym z)r|%?4-BVA55Ddx)~SqmS zsXnzHX$5rQsxRAe7oA=m{TfeeFldow%&&4+OEP+xFb9+xGqMHTi6Kavam1z*WP7hhvJbRra6jf6qALWZ1tw>*g zm~|y}qG9(_V}fr$E&ESnSgxP&;n|mc)S5m82^d|BKjf7>bMwN_=I(hJbHbZW@Y9%Q zm^|3bp~z@+m|(?sQD1I0FQ6Qnejv{UgAwRrS zJz_cRPUobH9w}yZ^TFqqL+{wWaIg<{1j6nE(Z{k{`_#0@_7an`&kv`mU>+590u(_H z+k1|OCh~pbOYt;E`NY;(9bbj~&X0$wEAYqlWm?EDAgE$^lsvze*C@GR5inG>R@ITK zeO|Sg|DLRdOH2E1#naA5Ax-;EOAZqyHuQ}w=9eesp^!)M6H+EB%4nAW9BgQ(11TZq zW==6aTZ~!~DjI!CSj4mvxp?b4ai91dG9s2#=;S>sJ=924U8V3qsFAjxZZPDUX7_l7 z|MIaYPwDc5o!)VK$-Dc5{goZ`{TvIX2&&8XZU5(X4xHuRyrA705)Aj;RJ&Q9tr5)| z#1$l4CjNPG_Zu5b6R0H_^xCg>w>Qjqc!~*M!LXC*x-PJbY6>A2*zuO@G-pX{mk|h> z(rPLEmE5*mP9h$xPBOFXb}RTB+-x)8KRN3Vbl9)t{64ROqemj>tC^VDbLfKSch=Dx zT%4Zx3Q>lyX<@TcRuJ~dQM89SD08+h+9*~Y%KF>*S5UTO_l=~-wisuO^_$0sW$?-oY|tNhg4Kz= zH4#mPYl6^-geymH5CE?jxK*XAkSwP}*ckOsx|@&jPGlfcSpt7Cc@xsel^8YWf~0`M z2MgHE>Y!rXD#1Ua2{pCz+mml-<>FJWi_|C@trNFjq}H#!)(U*TMsO04&GEh&L$)&b zcp_g7p(TEzQKbZbxSo4x@ZJ1}{9z7gd#BPaq}r}_qS6E05{7tjZsM$CS&E=3Z2AGy zgONysEQr>=6A1>fPYA1}o*N+!v40~7P8tPP7~#uCYBY&m1+OZZ3tNtH!?0#d7k2#* ze*``C@F4{D=_&4Gs`wiaCv04#gAnB{92X|;VHY0uF|K+c;(7tl7?sd@PL~ic#)aQG zlr}_k`8D5b%c%-YIN%H{nj-%OA>RcwbY`W-Plbtr{FA5v4V{gkdT1r@!C331DAdaW zab55sOk$pEgiU-|H1J7H(?~_7SHKehqPV1{1+GM7yR$3aOo#hkUOE%&rjGvLTZBDc z6^LL5V3VOgi34wpxR!ct$x#mVS5c?E<~{Cap7cJ#*AL59omd|tYuoo6%8B(Hf?N;W zb?_Jz+)e~s0}zf77b#tmxJyMUBO3Re84N@gV3PX;684)L1jczGgY!%KEy41`Aa-C* zgI1F-vjTnyTFvyI)y(`^P4y*=zsTBBKmP@B(VZ`R;0f3gc+D6kcnuB6GFh5K?98%x z;Xy1nSVdLwnfLHh@$v4(S$Cv@J

~I%F0AYr}zqbY)4Yp>Rz%B+NG}RQ00GETRUZ zLKBq>Oi}|q25$wrbA!iw=QY!m|pLuNuTS{1c-1ex|#jr%Gu{Jtnx|b?0(vb`vJNZ1d2So z2M9L+tu2b4CUBD9FkVD*WgndN10wQa3cfbKd#pIIZmn=!XkuRVF3d?f-%Sq)X%7JF zE+CA*D~6Q#zwS*4L! zz@K5L-{s7TG*nD8GdqX_uXPJzk0D1pqijn!8$LvlLd9E<(PfEwCI@vLr-Su~6-bGaS_@eY|Z&)O>=U)StLdW;0*y3n7>9 z4}M9Opq+wCb5Z!2QWvf`-P7xDrz^JeNDu_NCaB9G$7LKDv_c0f34f=a*+}U5%-Ymmc-Qcxfg z?D7KSv-s?m<>B-29FhQA*e4ij=YK5qrl2o<+G_)U%{eds-tRMg2 z7B{O`?#8U4l@`V}bs2t585cb!l)J1zq zdF7#K<45y$Q5aiF5JRE#?+gWdz{EuT+L!y&4S-@b2Yf0Oe2P_}9`T^^3ATW{WOQ0>*_4GZtPpb6yr{AQ6T60*@`vfXG^Vr77aXS^7(HvZqCS}R zvf^*~)mPjFO=+KW%8Q%Md1M&NZ8{XoYf1{AQ#F=?-D)Z|ozLG+c#U?M8tWaGx$>7f zOdzB}#lqEqM)5F%=m!YwuXvT)*-{Dz)xlh%P@K5FxULr~5te-ybCw{whv z9C--6r0$mnfXI->IV>UuKJNM;rX~KJD;uQfr;cAoG@5GvR`k@7+qA_0spu#ErRW=7 zdH*+xUhQ9s-Ye<o`e=FD7k2VO6 zgAVc*>2Wn1Y%sAJ!#g*j1@>O3az|sRp|f8Ul=DWE6*3)Cx^8V{{GY3s($k!#=nO#C zN1Cf7$=rn+&j_o1)>OOcQ+3XmgQEZNdhY9waRBYVxgGZfAh)AMio#To<4_S!Zv)XQ zg)}_|f&_>Z4F4~;1O5Exn~MYbhH54oJ=I=EzzkWZ%PfVT8*5)DxBZ7np99NuKX&4n zHp?sO2jlr*79qyC{hKgRSr&Sy|B&|~$`MQGwM|%Jz0zeXM}L`>1Rdo=!S-M7gO}jI zb||=G2mZU|@g4XuIrZF$le6)ZzX>{5E%-j#9aLC*_c{Nb`*Y8D(X!u+|9#`^2kqqk zrQyXT8PGniB?BG#sf+N9=>ZWKi>OHF)={Zt^GDy$YuGhl#ZO5gq=H)RY50ZO0Ey=6 z#YzA!GTCPQxUZU^11O6?2T)Ql7rI|kniWvr5(l!_maIX))o67SR)urW55UVn|iO6mJ|gX@7)VE4+hucG8IJ=v=WNb!@>gw z*G;~*Ii@oNgSADSyw{K%ie#13sn*&!6;3v@tEwK4E0wF+GFzx-oC5WsEyibK1Kt|@ z88)@9YkWg51b!5`{1PGVVUB*Ixi7cZP;{;CMfofdZ5#?x2bQOKZBO>x#3y-)vnr1U zMq~AKdf=Beajyq}0ML<2)Y3RMkzJ(gvk_9N@eiXGI~!O8K_P0Lj5Aa=Lq3;l)$`K6 z$V+|arcYrV`YK1Xhsz!)!vn5|6%)!z(>S*WrpI;tZZ?@QxZCR9!$Ya?7Ta(HR_ZEyMznwHPKKoby44Usk2hr+cb9A4E zMtB9z<`SGaUitct<;s~4IRwhK*{wvWe;y}3-nc4Dtd8-OeaCzYV}rZm(Q89`uh!e{ ze(Iyp3q`~w9%Sh)6nJrQ!bfFyp20mcp}7)PmyD##9e^S#Y*7ljtqZ_cBZ$hQfRN}l zZ272uXviaWbI@(6w7(>TjQbLF5PNeS>0*ZUBWh1-*ZpEbSPf`@+ z&7^P$&r4F)oWQI+bB`5#FSp9WEHV4k!SJ{dwW7k>Z=@aZ_1_*5ap&6F%r=5_SS_uzwK+^GAtmk?^d>!seYOe{FuF+zPBBE{)PO3D)+gb z7whu#m8*>jYP<=MAd)e~E5)+U;j3j*gd{rJ;db``7MqIDK7*w9P`KA^sU){3ssp+- z5#ydmNK-V!^XNw0LpQIVlJ-^mUg_1$g=4zX7E*SPbao&QN6}BCL#V#uNuzu-f3Ze` z9l%w*r^YcpHcffxD>IWGg|?W-JBfsi#dD6n7Y9et(d6rZ-n%4mTa%)CS!u1(7R&J& zi#Mnja)$+Jeg0uY?{zGI_!7(KH^cK512+t-?~PJc6DW-0NBAx|eqkV91K_8^XTci* zxNpx-ZQ04-x$iY zN>fqr6A{o5U;1~|y1JM$W3EM{FR#C_BPYDYb_B1EoS@2&=*9YPyC3kT?+{QljpvRm zQMU9uiC^wO2SEC1*1~o(1Gk(q(*1pf@oJZfHU%HPAwsD&u)MF|bU#k14z5$@3KF{K zzHc#AFSOgnd!L=Jcv()LZL7+heD8dJdTL$8+Z~&E;-zR7%!Yn{BW{ni;`GNN$OMi7 z=_)XGyd3rrvb~Ovrhb35(|sMv6Yuc(LHizq9i4td#$sZvcAd#wXmXI=Um%Y}!RVDc zB|_C@TLp1_gx`7YfVWO;2{+Z&5ZEr@W=T>BPsVGm%-@mNhb~XP<h4C^%%0Xhw%JR4Y^tTY`DR`1(a9Y5FZ#8?*mnR2TQ~$2=^e_d8^(&n zui6^Y7tQk)*liQh-JCdVF7vzPS*X{jSJ5^#2HqnY}zEjV|$Z_-@l&({w-Vt z-CO2RuG%CHUv;H$F877{eWBtaBgF9dbc03g5a4>qc8-3G^-YKD8J8bnB>BP*seZhF ziHW%}OxBE?Bmc8T^N#jb=bx_A<5(l2jW=Z0ky2DV}6ke{lNra$ znF}mciA-Tw#3RXp-1*2x9{$kz6PfOk;Ip~KH59ki5G^`QD^jV|E0xskIJi&Aqd`aW zq2-Z6vI`3fh9qT1dQtP5VuykKbi44_jUVNF;r=evYwAW~~1(S>?mJE!8AcjO$ zs$AvFi|qh9aq4{W?6IF8L|7{%K%ab(NO%rp6mPqP0ofr5XfJG0VkJd%eNxgLUrBb5 zGickfhAxNU-*5v+icU;VeS0CHZhffu%uSGdeW~#?T*&S=xPVH-z-^Kl;O7{B<)bfW z(f$Ffi_BBZ0!2#gLcBKdAUc0TRukauzpZD!1-2Tk9DU4ou6d$E0W{+$h z1%DO*@f(Zh?)*K#=2%mz_wGwXNJvsvFbK#k1f)PfB{(dpBiR0jydH>_e%pA2%eOPT z`r)vK4QF@wEmhhsbAcQ+p1F> zOMwy%-u|NXZo)0G_V6hIJ7+KVn(qga0tC00qEu(+y3(R^fX3@qq8ZoZx2@&X^EsjA z`_HfmhpK$~ss-LZCN^K2Z06;KW>!xYxq-wyT4PO5dYkTkYx^2T=?-YeTPTdHDz*GA z2nksic4oJ&e7=ovex>wA#~5#d<=4&LgQ8!BI+fT|%D=8X7}3u6h#?b#o@_Bqim)PM z?MZYsN?7e-6Q`;>PVca%H92c^b#$77^DU&h?PDdLJqgX!~v=#5C}~ z!(i@-scs}kUFW`et+qO_d^TkC3vgl%+u^}Qf{fhzLDn_gwjE&Rb>=n>lo&J}WFVIC zOPm+PMokec+jok+3y7;OVE{g=ssKIw7tg3qfNjC|D`3aKCZ{S5;K4G!#=F=aiN<@i z$`j}Z&|Yl9@IC>4!C&P}JA7~VS>q*#e0cHIC9XQOpytxLvbZ-SHXr5E(L`Sb7LJb< z^yC)!kEfPZoDcg7Twcyx?%gkW2x`&rUQF@_+)ttlbY%>lqfBYbcGrFv`KR{D@#;Bh zAc?2Eu)&>jN~B{1CVx-b(RyUt^J##^MsM5$E*F9Ri8Km6)mLQ%_8(8E>w+PN7F_$g z3oQO?)~C*vB<6Zj&Z*HKg^OryD~(6+zeq1^P+_hUQD)W39VP;IVM$;(=9$G458yo= z4lz#jE`+AE_LRp+MSzwOk575Jftz=ES45SA==cQcgvv`l19+=&j=KvIda}g9<*Ic9 zj6}I@>J9kkBg%+8sW$J5a8ZBOJg zp88a&t;JGZgehHDx6losclLjiW_+c~a~3)vV!g0DPSHB$>WDvQcu?%{-w?g#2tYm@ zUSt{Em2Kxo+v0L=6>6xQh{5@O0YOkAuE;%aCZTvAX2^Wdn$)E=+4v(9$2&{Tsc%TL z+^5!a{e?g;S)&6dA*q+S`-Fc{73k3J5f;l41&lrhwjZ3`d=}9J25g*UOS}QON#TM zHoL}6nw;qKmS8>VrdzfvtMd3*6%vP^I?b;8CmHwP4Lx7k5e36}@-NBJOnYFCAVMO+ zwZR;r&5&y43c>CTfwJfj1O7%L&Yp3?gj7laNWf3o(nM0P;1ST6g@zC&U;{=Hr@let zV2!3CR3i|muuw997Zb6C$5_9ZH$tTqv^$|u~X65Bz2%-^Q zmmu$T4*W?}EfHDTu%pdW8KWLWf0cZtGnUq~y1T$l?K@X}b@}MYJfIQ7Umas9mro)% z=E#XPl2@v?dzpV+ZvKY#wHMG>$tuw2f0@@qWjk4a-KRBO29&tGc=U3|c#nSvuIx&7 z3^Z%S9f#$S>qLr;NY(Fz9cgl=6eSoJ{pKf7+rLo;1)v7|09^M# z*gUGfZZ~ql#?$dSs1w+j3?)*^gf&HJDm;MEr~!7HF2?xF^rObt0gCL8zst$x^D}M} zAFKz1E^lPM{=ADAFF+5QT9O}K^=(PW<}pSodHP*%w&d%d$)OUS1h`1M@VXgE1F%z>A{E~Mg#Ys%%6ma4t-$(>K+!j!41MeBzvczmCXuFx6a02bw49vn+aZ9m$FL|14j-iQ%8zQvXG1#!N4eTTS&;GoAxqiddY2Ado%| z@M&5zSiDyepk&kt${5Gkd#)-S_%nTyE3d4~1-@x76y-l?H%A?hi(WivUMLL4dz3wF z?eXNFF*@y%ZIL1^5wGg%S?MuKQAJ)HlM4uIZVzki+v<5tDMPpqLXM>!- zR@{{6y=d@&k4;;`snHAJBGd@95ki(b-?mtjo>FH#m-h@muhYN7gR+!;z(84u%OT1* zzgPmwW;&smX>gU{d0JdjZq)B2?x9AaxtvgtO~gpFj(4aZ${hsrX$hx;9l=5%)eOLV zkU}6K@QmUUUDZhLGm=P|Z9Vft>0#CUbtNG1{tjD_{UdCJk_kewFrWTLv0^{|g<^eV z{~N_JOZpSGGGqTcY()(UTahM%v*VE_!@oeWXSV6lCybbFpcgNlS$eM%V;zG>ezf>X z-^`%$=BN1!;`nZcshImyMQS z3_GMAVEYAh!ME6IL*fy2$fO$SuRLvI`{kDfnFy7_t7tv5jgG|q>dNOdXCuA~5T;(L z)a%;gYOVjVNpa3AnojJV7uS{Z7UqT*7gs3_G^|!4Vi8) zCsZ4wSeQ@!iP1h)oifQP9Hn$lyIEf0bzKeMUhC7;?(KfI(?7O7-F;%5Ky!KZd+`~6 z1^bhHb!y4NId$bdP`-3QTCNc`M^kUadP+aI0HP~!FmO;b@sM63roUG>11SRuQ!I_p ztjD6qBH#a7u0LkTbkvC7i2tq{^Vf_cWD{gl1I8L%?;nXkB4>p@pw9TQ; zYZaQ|;;nX|Te4=~`K##^4&1HzuDX4q9a78FCzl2+v{}XT8w)n(#{Ll==7v_=CHE90 zlv^`&R^6u+rr*q>RpQTFrT~O>N(p<6J#uUtIMMNgJka+*bf~G#DG)R$Qfn|-oapvc zGG3uO5|g3UqJ!A1EDt%f+%chf$RwL26=))~^gK&(7Se#M3;bQhw(+AC-~Jl!;S&J zUai}mAj=1vBOgM&mcV6*0cVsICb_sIIiAVfhju(=r(4IvNsFiIW}~)PaWpx^e?o!OgEj zU4f-V8aoXPHID^Dw0r=#FcO+#!SZHfLh&*RfT4g)O2R+R6X9pjlDHms0ASb$dpV~z zlH`UE;^?SJfe9fW(Z$i_00!Zdf#P^3=%~qCe%d0`S0O@HWRD4;%1)KIibO-{5+4nR*~xC{@SU)zBux_K(x1mQ+qn=y~D=-7xL)z`BaL6GIN z5dEFFvwGzAO7<2E$XtQ`r@69U%_i~<-H>Xu@FQQ##3e<6*Wa)t%SW^r3VCBu36QnI z;1U)d^A!D;wK8si(@8W)Wg1YEaz=-l{xRZ$B&CWu2u<)_R3RwoA?xZ6VEjtmrPY+I zt0*J?Wyt!u&f1Y)g_OiD_WvCh$@phnq>m!l3Pg-g9Yo*~i$zN<0n7IC##`z|JvQ;7 zzowv@{ts&f6J)Kx37))EhRLu|A%iP9q)NfbD@UD?nm+5?H7HEn4_ZJfZbG(Ce6EP_2iN-? z&O|dQsaho^XkEzhCO@b!vM`<;Tur(YHz-4omI& z!C3r{Oo@f!f6J7tw*nZ1)@G94k4>Kqj;Zm6VjsLb7`q!?o~lPKTGl`j6loaP*#^tA zTXjqNQQ)fW`lrmPba8ZiyxmMOhrWmbAF_CKg&^!e)W+#XJAMxNX}!L;&VX}~ zF(r*mfIcC#$E1%Zt9@RMK^zOq$Y6=Qqao29%x`DONr7MxMafz*qK$$hbK@l;Df-#= z+!0Ws6uHJ~r%=`eifpAtc%j!)eg4D{waLjpp;YVUvCZ{s3yTxt;h0N2;Fs?}DcXO6 zl)7zw}4zrk&%!$b_rmm#Gbj{#8RZO4T&)>G4zLhM_SVY?Y7K+Gcpo6C7f&!qB za4k~nYdK@fvl{g~GHnm|0&;3PZ1qP=5Q&~Vj4_|!1lp`TLeB+cJlaYO3T)Q!Pm=TM zT+Gm;-uPY?(4zK}8N&=b6D?>j48wUWnTWKT4{_-Xy<%$xp^3s3F}mn*GFuR-y>jW`x~Wv`yVJJZqN#-9Bg3y3XT2bB#I>(J36f8QdN*h z@m~f_)+fP3kU@h0Ea`O+%K9`Q>^Mnp5!s3bVkswX|2Qv6TxPj%*TUnVXo*o+mCUv9zpS5^dsa+UAm7`}Iy0GC~4fHy^ZTQn!9*998;! z33Djp#<*ZtBWk@ZQ?5E_hQ`Wh+ZrXWuklx$L_CaG)lLB;J4t(EoUq0QbuX=_pOnk#sx|I$5Eg%~$ zDi^a1x}UJ#j^k5RRMRj+qS#jedr|ePX`*6^y1AV5DR!Nuj8jvPl{S7j;8BZUfpb`k zV6z5-#>k|~Q%1=7vmd!j$OEOO+CIT0Xrk466zuw4rQpF^%9BGqy?`K*jmM+1p$X*Y zVNmz)%_svj1<^-3r)td_f-MB%@yQ`-tIl3vGsoC$HN3Z}&7K6XZ(3gI`@E466BshB zp-Vp~h#SgTcucLV#>}z6?J1cb5`aADwLfnIS-Zh5IW`(wNdPl@8m=~@)qPc4qpIS7 z7Bvhysr&r(ZcQ!g6Fc?ag&c2-oY~B`btVgGlpVJsmFa|LExe)pCsA598W1 zviAKcd1&=D6meM@grlHinzVIYhEW%$b05AhY>bNP79YsiX!G$>vKE--=_|6QZ<$<1ad>$j?QGxf=0@wlN&gYuH$M8PUY^m)WE zeK)NM2xL#&Q!9+L_rh!#Pfg;eYBW<$Z{mY^t;aB^kb3Ta3^Hym33i}cDl1w)?cTYmT+E~kJ z;5Sxq{z?9cl%2Q%OXDNl?jG@>=l3ggK{(F@EFszNcRJ!HF~-|T(O z!-TUI19wItE%y`8#x9^uu|l77C((6PYNnQmpWZ%9Od>7C-5d+@b+=7NPq*VV{acib zH+5ST4(@32Ns*uO{$~!kY=EL)?`^D7Z&0M*FCrBY_=UYN85S!KYIV)eOdR=(Lu=w< zGzwORL*A&b-f%7Q@8)lMagUKlke8dIP{y%h?zUdBUlnmraM8oL!UVtQm@Yr#OMI;i zHxGMe=D@6Gzcoq)*TMs-T4;MJExZa$Qz(RAo;0FgzuBv8qg&?pKqLGLiVodZW8X*P z{w3i8z8}NiuyvjBGi?Go!VwV^@iMWN4QW@81~2U+1i`9!TYVGlW~6 zH|vgg&P>+=-dW-v*pxlYiMEbE#DJks$z^KkYG^^>b(%UngrTF-%Tabk6 z4tVQ?m^P;vG{Ua^p!Qy#OjNq^nBewaR%SsLE>&!>Cs_hgw`t{)-86?>O`P@(3E8Ps zT^DjOch!3VXSTJkRCU=8piX%8vOhwPtL9|0^r&Mnh17=-i67rP*ozPei3X&Vk;=mM z-X>9yXmDLia6SpU5a1exKTf&E`~Ac>Gh(cDmtz|b9m+BcbbX(c)Cxlc}D zDb;de&jIeDZz3^4D!@G9{CARLpv$Luz&j8GMh>?{)A7aCPcVu5nG&}N>i1MahB3|* zl5;`)s~UuOau_4GAucUx(VXri^{0;Veq|n^YnV3`-v{T|;3yR^9ax>=((0n54w#ZK zlyKOQ9>E?#XM!^lMDf;M$WzGET~@>^ZG2(Qt|15u2#cgODkZ zhgkiCs8)XjZazrh26li1F4+YbDU0Za1`e1P_%%r226iOiA!T^9q8V)%m~5nGjDp=- zY!U@-)Omn)f_Xn+-Tu*V$KJW=;aGPcmggd03>f^kr>VcBhv}MscWFAPBg=n{?Xm=9P z88{dXG&tB83Rn+_;|=`7@do}W(!dhFd|Bxt)>?bu+e+XV67K;Q>j8&AQImt?r5vmv zbPSxJAUjW(gWEyTj1VZ+UY|j3R)Z6!geH3=6(D_7Aw}v97eC^MA^D91%bE@9R6)uE z$Aw|dK8EOF4KyaJ|6qjM6amYby#zPyh&5ejhD8hp&_@LWoPvq!lOhGSfC^3mJwXL0 z+Pf~k{=F5iiNBw)lylHAfYSQ$BAR75z(nC|pXoXuoc4V2E3*|n+V|fhjc`<-`{zYY z(#Si>sY*?aD)f@6K2+nj$&`GSwWnv3{f4ob|C^&Bp9ges9zRvXuT!plOZA;fs81<$ z!_9d9l$^?YFzPWIds6g4PLgynUuc_lZOkVx6W9V&LBZIE#MYC-=Ass})2Y;YAy6MR z{MB1<(t;Fg&@i!;oXYiQjxJCm_2$6Rs5^V*ku2SWjv|RxtTHshZ^#-p>w22oEDW4@ zK))@4A|)o|yV8P@1scpVRM&9fgd# zOLr+h5nz7=p-4Gl-aa!TMK757#RSXEVw|#RcbNX!cyasX-1R_v$@9VLJ9Mpq=LjRy zG6hQ}FNPEjwi9P12hqpvWwv-2hQHbsBvKJ>Qs4gp90EcCAmC6U+=lczBWE_9`fQA- z!kJox$zWRR8C0n7bkxwK|EX>7v$l6r5mo2q@NN6NB6|$M)6Kb#zsGmQvS z>@rfZ<<#m?*`!7auyG}+kUft@OzQ7)1u`{Xmfxx-->IWG-#B76^RYpsj-@obx56Xs ze%=e8qt+pBzRB&5uv{h}8y?yaX{e~leWjJs`rFp~W5=eziK|bsOU->-3yFg>Nm%aV z_2i#eAza2YL>w34fAlM`@q%K7^*;(d#s3*A+!br)eCf4`090;!*Q{oE6+cBNQh3F> zx33Fd+${sw&fJfI>+dVN5C7J-Ky*7fW%xhY!{)O8#vT@7JnL=Xyc4AV-L|kOSvI#) z{(aSgfp|CemX7GjPRZ5#|94YDQhQ`j8u9}7L}tm{@L~$Cg+z`?x;N7?`Yb!ZzHtB^ zhTn8T2SUasYN5TTR<9heb$(F7a-Umb95Kz?<{cU$AvIxhx|L{us6q32?jcLO{3)1t z6n4Y&dl_jsbfnFZX$TWC(b{m>@#)V~H8a3@KE*9$O>)x!UMt`A)^om!i zAoS2`LRD}~?qAAwHA}z|q->wxOUTZj|F7ub#2@sK_~mc(uuM?y7W9Q{as7~E73h1V zTsAsDek-HcYJ!_xrqS_1O=`6a>DX`$+s%%|)RFfmyGd(@tnTdv+dyuVb4AO2aDX>) zY8>_Mf=@d>G`8S;(F$YKg{%8c;zuHv^Z%L8~Xl8B|B*pz^oJurOJ%20BJ|Fp-=C`k=mRo zJB7Ib#dus%ncm$>OkZlkJCKCbx6iZc=# zWp49ZzWj9>Xj%>e)nbmr&e_&G`yP_x=wibdrl~UVjWUs$KLT84tN@RmSwBT?x)oCthLD-k*s!yWtQ1@QZy27ryoX1>q#SX5aA_Uz zFa!Sv0u+fDa$_5LvA~l_8|i=MXUXuXy&el9%2SF(f)N6#>u z<<&enq{u*qn!?Yj^7o+#^uerMDKILTc1Dh81o;oVy%-ESWaym^L%Wy>LWcl5QZCU% zm}UhE<0RR`FOg?x)RcDR@J4U-uN*pqCa;jPO>0~P4bvSsBfqrA!$17w{CHk=d|u*j zqKp`4?H70Ksz+P^OKe<8-0RabsnQ&dl8br~wP1bPsO%Yj6cb;4__z+~J{@0eI86ZD zR<_=w&}urypEP@V*C*TQs{3b}ZEer4}^f%5i<_%XQR zZ0~hSgMh>XMc7_ojw5cw0A~(FUz4keCx+PD17|_=5tvJ41T9ulaW{2F2+{0!& z#pX#C!3Wii8g5&Ngsd;#xX&u$q7=UR(UiGI&km}>izAs$0jle_WW7B1BJz%qcS=To zW;o4ndPx8EPi`Kckpc}&VJrFls#1sfQ4jCpR%;y8$Lz;=V$EWm%RKNksH6ML66M~F zTOn4olXN9U&r(@qiD&^#rcq}#rim+)m5LTl{)#~H8Jj5RR_P;78FL2su~cO ze{@v}?(mg@zoTEo%j1NB=6y|?@ndmbXY4l4C9DVVtEI`Iv%H|9$aNH#*Kw2|pms+; zV81c?eYvif=n;aVl>DAGad_sA;gGK9e&G6DSS<4PVE0Q$8|l5GB%YWC8|vh-6nKw* z@oPg2@P^KX?UhC9ClubBi!KTfHVrU^aqA^4tk7-QdLpQqb1>WR>Y7GZ>&hwR*+cvG zCeZ;7>cHwlU-0_OuOMNmd%!p%=ov^F-C@S8%9!XY3@nzk$o@$Ryj%A;q!BM`3j1`J z6_CFRysSM4piN_}xt(&nIJIRZJ*FDkJ@@q8m>W6<^B8(-7;9+Ab%^`A*Nl^goj}rc ze0BPIF(Aq)_s%>H6h6s&m{fn-JP6*v`yJr0OPdJHdF)GgMIPBqcs0m)%?Y7NctXzH z%#@%m_PZeO)5xs4P)ddTsLCtoxiMksmsdzC|2q%my;D9xur-PJ1IJS>270A}Wc2-) zqe^zZai1^8VvM?t2F^iHv93nXT?-uQduB7vE>lszvdIlEfY--bGhD&Hd12t6Zs>Kz z-g8)>i^ zhYXh@jTu#sz{lQZREz1^;=KW3*20srmf+rX`AQc z>(k?+IISa56HmvIke^uyirKAfWb<_gO@G;Ax^)N+hIY?`4v(**&O@WK_WhimlxzAc zezy=P?mx7&{#u~(loneBSV}+g&a5{ub(80OH_N2k%A88zfM@gr(zgdMV{|Bq^tyKq z^XHnMqNDpHo!n_$c$s_=4zT*7ZE0n^x%Zu-cedsFb)zZ#D?TgNFHHwCYHMFXRdbO>{(MW9O7p6aVR63kCN{9WR z=A4lEvB&}&Q#^6dci7MLvfqD5HD1!pnf|%@w0&0n3+++Oh1n{PcQ>8R+hSFhtKtK` zxc7DmxABcijlq-b##G3n`SHwAAP&7=#hX5lMh&}wrMXAwuNI%Rj7)-`ai)&2g&vv@ zqIid}qcsJmZ58HAQ}2RjznUM49E){pmE-+c`~$w~;cN&x9;1w9rrp>{+7pegRDPO# zZ>Ub(`*ct4oxDC@FH-&P!%YrTaBY3DN74M=zIJvuCGh?EqQSom@@{jDxBj^M2}eoO zr~R>D#HPxU*hzD(aG>zW*=Fs^r+(d4%MuTJ6%)GzvSr7U>E82Hosps{i@`A4fcdnZ zv6sQxA5QJ;i;)FR;YqD8zFR;7dnBn0?)Mr()q z=w*8;rnoIQZMyGNHi`!5TDVfCt8Wtl6T&Ciq2|ljtJovU#Q|O%4kN?aEp7Khlm6c3 z75W8YL+9HcMzM~~U+?)W18#>MD|ZXmOh2)#ObaQyP;Lq|!TJxGRx}y6i$t=F7qQ~; zOnu}{7+~(z4GI_+tcWL(n89o0Xg>6CGf_qB8O3pntJiL|3rlrKm$QyBY`|EqVF{>RqwqZsVRY=|vcTcXL}H;LlJpT&HKc z;;Bs^d155Lj7r+RIE6V1AVFpge}(AAjha8WCR( zEgGRD2pgf0o^@cm1V-PDgHxsl@_~(mBYZk{N6BBs9&D5PRWS+GM-cKPs(AppRDjZ3 zMe5d5CA#XxWC5Ln=$wF4kv0&z$pH`cb-}rEr+G|4L8U zU#dA{-(JA@Ct18Or))w9F6?aW()!Q*EtiDu4>%Eh?)1E?cbTSfJ2t)?-^1~0sp7*- zeZo8My+Y_x;+qjdSOIBtDk*rF{%-jp8ASX7YPy$~KXS&dzsZr^WxYA$sbN9_QPiHY z4ATWG@3{nSz2iS?34}$uG(3KdvhS&KUrfd6?3APy{(1#d0Bie-#@R1TF z>>9#h;+v#-6`rq)(kG5=#^_Rxc6@=ji2YvQ?4iG$qLu@vHKnmHBV7KKTEm#!z%|>K zVOU5KHuq5;b5H(E_yVJm2{E9jD>fN@Y*HUdDlD2nLu5?Q4I&f0rBBfqNp4Xuoa#G5 zx(v@$A3R}4FYO+%SIJuRZaE{3@P`^2qBye>9OA{{HaPVAs5*7rR+y6Ek+UM#0jQ8P znCC8-b)yW@f&slJ{8Y3SppP~-U@dW_7(;I1PVAv$+wr)r%}!|uitBAs%%LoHS`me1@(=O z8#tA{6(&dN5%W#(=!|@Xp>IvO0?suY%rm^K-y4Jym2b0>UX0yWTpn>L7Ql}f{KU7j z@+|_808EA|bE=keUvR$pkqn?V>{Q|UTEN5fOUVB)EFoK$lzaAE2o`)>^EOze4 zwjGxc?*jli>6YzPif^``!Q*`tKztcn{HP9!2I!cm%NGziuq;?E3~!n~xq?I=1OR;) z_KFSwFf}rOfq$?Ej~O^d0tdj5BES%5n$ccmH&A$-wK4=bm@G^#1}{TVL%(?C!ahmJ zg?)RZQmyoNY5?H663W%ucZcX5vqxsZcABXs8_?_V)(hZzmFwEgo>aZ29q2mF8bage z4zMbTDWEE7?O1{;{6d(;bf&GsrV`@_C8Px9*s52{54UB>-dNe%V+lfZWf=h8^Dp&Y zCW3+NE<^xCRB*&rFPe2hp(*jF;qs$Ra`L2699`WZnBsU-C_bB38?OwYO%q zuLdnZ^N#_5=79mg<{wx~5Y5!{PMC&_j$wHo6^NcHVCf40K{!AYu6H$8-n^PIx>~@w zgM@-NIe@f(y~Q1+Y@8!H%c`LGnn2v5p@7_>VZhv?%-_`%s(weTU^xD2Ob8!9uAvf~ zpqwCdhyWk}p`Fo#Jiy%N2X6q^wI0{k`|-D*Zf@r@Nt+zH;5~2q z0}WW;Kk@hs7CytBU!k4f|7nlLxO$c1*XabrtkAt6+MMD_xV;@>k#`b$+e4IvPlx87 zpJ4l8h({Ta2RuSCpp1@ypVicUjtvre$D67aJd54DR)9dB+kZT;d!jrI;AZ8gSu7aQ zu4n|-8F>9XxbGo=9V9>BhQu|Hty8P@KC_r+@u!9(f;SA=qdQRc-n znHIsTb#I5e2CQgifaJxu;Kh%&s2I2lroDMs%oq5p;*Sl7OMdTyF{9yiW(UFeorl57 zH;rrJ9N0oPS`Xi(VLEm5>f)ClrgxLcvs!l*XMYN{ZVwE;@pBy8#5FRk1#`cN1Vk^h zj+60jc<+*Zn6fIJubL04@yGIaBO}u^t6u7J{{Rn!p3H~M?~sHI6Uq0d$E==#?loke zRhE=vF(#kYeQ4uP`aAeRy~CXESD*GZfG2IaoGquLLy+uq-0+IeujOmTC3*Jb4Fx^_ zT2x@Gj{k{zC*1XQrNeAJr2K0 z*c+`Erux*~R}Vz0(L2p?l8Dp(?-)J0>|Zf@iP3Ac*+ym&Bl4FCl*1^Cj6ms8YL|U| zmK-D5LG=y!*zf4}Ugpa&Nnx>?E41e|E4Nnd`85Y5}Lcs{t(OtOHEd37^Weij?!b z05=BycNd=eQGbNO4?F^@YFveP>hLH2L3D>|>i#^}zQu#>UM3TD*&R0(_oTn|0NdR@ zW&fiGxG2xIvyye(hD?&$WC}xF)mM8-PhZGOX=6y*sMZ;AvDEjVs{aVv6W0lMN|;}{{;s~e|1!C{$iV(?XVd$- zknyX{5SFG6OC%Y8bo7u}+A2)j%CK&4ZzY?a#ziF}gk*rECorw~qr~&Z1fnA-MgVy_ zm~3fV9F@WWaAR-ld)5aSeK5N9SZG7@a)zuxOA}Owd!jL(rZfHSQAbl30`}6|rX6-5 z^eYKnN_$UQ<#R2r^Fm@fJ2j$-Rxy#MG+aMzm9R}3NI#ZR*gxK`KUHWR{^g(7?L;#%K3ka_9oYATKwI*~a7dxurTy#d@?c8ealS$dgnKN0rBM z>)h`eaS%iQ<`TVz0Ogz? zFfI;LF%5)7`z>p_y^9JOXs;_nfJy8tHt&)b)dj?Z$eR{`vBUfkCu0jBydtCkjx+~rpeS<= zHg*AG7twbB5Oyp%(li_oeLIG{VHWjP%bka*RwNgZ;0moy;`Ak!Q1|`8$q-xJc%J#k zPxa)laksM$r!CGD2^IC{X3Qn7KyK+dEnkhZ>+kdekfuCi-yfG5o_mClS*XuF1YgM- z(7Eb&QL2QPE@U!v3a?;De84|vxi3z-zQVs}k{(?xP}-_hqXSe#Zu&TRT41h8fItVp zO33zP&b=T+h~F_Wq=-N1e+1gw5wW(gdAb5*enN4n1d)vtf#?8`!V%#({?Qz4<7z>e zSIWn!L8(U>-NVUuY?=#Q`V)#ca=C~M#h|MSW-1ua;m4V?EamM8(&f7?Fg9Cxy!wvw zf&A(%-{$&BbvjW<-0Qd|$?cgM!f`8Tryw$V_)AZWVV>flnLz%CZnCAY!DvX=Hw^z4 zN!gj88llJ1?mKszFdzO9wxAcP^@&UYD1PVKwhF>C1Ph)%Ni=|@QDEP~S=PY^vYrTm zkGf07j@K9L2E&o;1jEs}#ZU{?YXFtqJq??~6L6(9;7T~mPE3yckZ_# zKpGkmHRd)&yhZnxYISv>3Nb5~l4vWeH5%=4PFNqCG2lT^!@>PS6lmxN9zTnd)-kw| z$4*(O0#U)fu|lDnh1QnB(Iws}d~34$tK4r)&4W3FuTNx=QKMH>|F3)(B%{$lf5^_a z)SWyao`ke^Q^`U<)>yW>KVotB;O`-#4D3naKZpYlAOUmtt&Igd{E1XW`r}FK3+g_> z)>sXM1>}pB_!$onB!a@5b)tBjrlv2dE6evx^lUlEJ`8l7regS(Il+w2&O2s~-$zfJ zcOf-duoU z-Q`{xFn(gd+ofr!OY4)Q1UpKj2|5HG4IEbXIiK7A@=tmPp98AJ5H3g0C8qP$Z};x< z$LW&EIxGaH!cxTd6oqQES{*Yw2Ci83Rg}lS2LVpnU_F;A3 zK)-|Y)_wYK$WpC30H9V0Bk$$9-c^>vK!4f_+2=Y=7`3jRP;U917qc5fjOIhqkVMof zhoxmDbWUE7CyS>nU{%UzcVb(cSJF+Hga@IX5lxR)NOHAo-VzAq^UrGcfY_`fpZiUCo^P98qvOiE)*WC-p{rH`vZgH`ox_!; zBcG3&0~wH|JN5NJCGyDmLM+dC#p-kBV<~uJ#-H_B7!lCP1DrAvcyQ0c~dF= z!Mi&!7}edm=CW~Eq_=eEHWE0=WDvDUNo%a?KBygG5RY{{Oq^C={wRV}uqmUVwtNC% z0}eB${JSZm;47b0@Ptv}UpIBYFa|P-0>L{>26s@i>>JEofsTtxg<2*cAOd?Ap@~(z z)je|n(5vVWwh+Q3d2>G6yeFs!nDnhv3}kQwt*8dB2k~nzQ49t4=K~}0UCjkd_ zu5O{7gk2$yC8xPiH@87i4WO9D35p^8eJ*xY+PbkSWF1Fr!Rp@KvX0M3)61pS-6_=j zg$;VP3u0n*sDlP@k&CpRy#2>==E?-sdW-&wlb^uVSPNF%F|tF2@)Y+z4nzLjv(l4% zk&IeJam6yEbEbSXg~8b#wlfPevSD~KJ3)#SxAAD>nw8vz;!z9u{eJRTuKrol0M)u&8q9&oti6QNxGZD-wDMrP= z1J96UF4+uON^@Ilo^^Wyah3|@=(Mi!Si4$XdNEySSa1Dhp0KGcIrL2vUjnPoWJgU8 zNBeX2%VfsaM;asP4K!P;pViO3B~xcA0e(?qToCqH+P(ffQT8cyoDw!g9?fqajDRp# zeElWSr^OvD^Lc`fzj1Z)PAoQKYFco0h)`|!$61kj#FZ~bvnaFeM3NbQ$jru__}>3< zO1KnT%%&UO?%t*%^6Dk}9t2P;Ajz}ze-=Te9AbFmsnKp(zp?O z?;%VP)*C^4vLl=LsNUCY-d1r_JCf)|ZR->zy5cHm@5sJ}t1&@gls`1;E{l!RXq6~e z=)aOcYOqHq7M89J*mCFYuIVDV{St^wmN>aiX!!Bd^WE@|t`UAWZ01s>bI?2=HB;Q) z9lx0GIMLP6aJWq?-NIJ59nBd|f!H7Q*M&07Is5gq6*72_G)rie4e7St;}ju>+Gua~ zZIpI|kJ~664$>UTDT7hO^D3||&*;fIOM7Nowf7S-JF$~aS|{JIAG%#&-`xZIK1)4< zzzHRwq`#U}s^Uf!h!PeQWCf66rRD;sJ!q=)#6H#V)Aw1npbk6UyEObp=B?VD%q`nT ztu3h`SQ5FFy6TNms|z{RNDnaRkShk<9pa5r>Bv;+fPRt>`#KOEsL~@>&817$mM`T( z{V?W-N|}dj;-Kg{tE~JLl%Dh`1DZOqzrc4uEr!}M@gs@Xn6wIvCOg3|*^J&vSRK;WDsIghxXWOfl8B9-! zst>SoJXrbY;pZ%ZH7M%X1u9oOrjgm=7LnN_=8@G5Z9~UMCn7_~zib(qscOJZM(;NG z+$YyZnWgEF=GnN$wf;Cp&AW87pY7cjHNQD|H?6eTyxgB=x9f{uyXH{^FFeV=>36W@ ziwvR{w8Kq1xVP(cVcB37Sl(4*g~Tt>$IocUL5)xN zFpcT$Gme_hXJojam&CBk1q|4fI{3|n@GZ(ptIk4qT5Rj zNuX`n+VvDESN3Qb20KZN;6!srAU3hEfwG6- zPMl>t;WSi4Vgq|1-lT0`Dk0dtBGiBC<5p^tmL3e}bi}|am?mK$Wh|Of`u2y-RR_6X zw{`#MRI(?L*A_8%GSot&xp)Zy_Z7({2}f%jf{7;Zs{;5n5?p!ked0o4EIZ#gv{p27 z?krZjBg7^ek?k=NxMay;KQOPgZ;#_z*yoSgaeq1k{r0?oE|&^x+{$ci7`gIrOZh=b zsBW$*lmpxWixS~uo_FDpnwqH6pg&m9DAv_|3z%=w(+StQA0-1Wh zFrH&&v~{pA_P4FDIhxsW&(~7%0t!imLTO6ggOw?ccO`_Z@ZTWh3tr@=4L;Z`zFY zk~5JjepU#YFT-2{M{^dLkm6z39PA?x!&5(3T}Ab9)P>6I{#W0HN?7=1An^3Au z%$f>)CPSdycnW%;{hlEUHLjp=Yiof-(?qtb{0Kw+d1aN^Om+C%M6L7vPTyeO6eX<1v4dX^{CKQTDfW>H`*n9$65|%91MY0;oeM84fl0k#2t8O zjMA^}vC2~1Fp=4bC@I*r{z}n$qZ5f6u9(U14`CE3$y+yDaa~QBFLC!*gTDvxuoBY( z-AR(sFBG+z-NJs&#hE)6>iv41YbHC?uYyKBfT!HxjSLAP;a}h*#%Oz|{3F0adaFAH zs`oqV_Q}!%u(5ZVA$Q0S=Ytq@EcYgxgAb1uJI{8VX)4&um4)*plf%oEqh8p!dHy23 zq=c-*ApnP+ARV)pbq?>7^wX}d6<@P) z9cHU(C{?CU4}-v1p$~DFCd2(=Ra!maRjwV5QOgBy_RV~O^*zS z>>&Xdo0QK!GLA2zGa`l_nr#n)V^3%sZD)jJdBo`Cpq~0-OL~M;L)lRf$lco`NUQdI zHt$dXiwQ&%SE$%8w>sIoVq0QMnOrZ2XDe*xk~%vt)G#lUDc@0T<`!4jcnY9K+C2D> zl|h_>T+~V6FxfMa%+Zp&ae3)7DiYDIAQ1p0aAtCi^n|(1=D{c8F-ylF&y=m{!wTXV zh*+116_N+f5d`|rR}Mx5ng%nhc&zWvy@PL=&ta22UPMJU(Y&g?OL%V=e+jgq9>SW+ zhQ1RYkBHYD@R=9F7KyMtNSInTre9KjOI#(%{VJWQS$L00>Io8A6L7ff_uYFCo!L07 zQ842K8VpnuZ7=ER9f-Lj0pc18D=jmSv+{-zE&R+xpDz6FBnolR7eo9{p>NdiGX=>L zFqJLTb|%UJ6@j4zA;Bx3@Lz>K>Hfb7ed5mluh2g^6@LG_&?lunoh@YjhtS_nO(F}= z;1FlohLl)iXKnpN6Ab?-!bErZEsqUDe(7Z7cP8z5cd0#n&%Rh;7%Qh z55*r~a8PG~1NBU-T_4n#MXdxIK$6_JQM|VW_BPUQx$Lu(k7!fqyUX?=kg3Iyo)^yd z7$B1s0g*5019ObackWN3OX*RJfE+m8u+(niNM{X{xRF6X1JQdhcXMV9?0+? z63?}v=x+k=AQ{R|sRThqgK}V6M>-abXyv&oS<%7;b%OqJCI&TpXKXk4JSO)FKgBI%poh1iAsx1^DER93%Al zc@FfLr6Gg6)=-^U|6h>w9wE|(x*ZZ0lW`8z4A$*e3CQJ)lqqb6s? zuTX8xobW~)BRu{Gp+`%OYiK~n9jWX@Qs0-<6t^{_$~D$qCx#r#s2(F z>|wV)G4ll?pR<)Vz7zK59|VXogRn4@7>57;&T{>(NGDl7Z5Db^P@A%m2NgSpg{DIS zc0-Mv`SXZcn|U#6Kf}@Dg-b2uFf9V!-7N$Y9mnGTtHifR^pl+`_xsO+fFi-EyV>4+ z02lQSL;aW%?;@0~`O=(jp@)P(^BFAiZ+!<#ePEN516GZ+t5C7~xLBLm>*H%xmd+9x2?W**!Y&!uv@y_Y&Fio<-}2de70}Mc6vY6R$}s@ zux@<>R^Aieum8Sd`DwAnFhW&;UoP{(6JvFllg3NDDz}a zUmLg3%uMAf{iWoLkR$vxzH(F4EYHK@C2kXw#TrEC6Gl!77=r5k8n)x_V^Y?ndVRIg zhHhe|QH>kXWc_`3fqa+GTN|X!@S3Ih;>5Udxmp|Ybz^ziE2 zpz|90%{dN#Xry1x3Ty;P8B;&`1JxQT9a3czgPE>uZU#yO(xJYH71GFFoE7SO5b>ml z4>=K`>})U?CBV)elo7&sQqFaLPr*9RYcs`(c|Wq&eK{zo*2jU4Vid+=FDI9rF{kW8 zsue9cCxVBKtoRaV*@s>P^)MlpT#%D#>hC|7fIr5%K2=}Z@<5KP*z#dYs^JbTPg?Aa zE&rN0IiS=`Y~+8?{KH6}vYEuFPrRAbXfCOl%qV`bncOHJs+q#5zNne9X!`a-{9Aj$ z9yHSw#q@3Dxo^rQ^IojwrL~Mb2&6TR{S;){Ek6ro94)s9n=2_C^s}Clly7B^uzC)D zR+;oc2|H*0lA~y(1+*`QM3C||1rkIi2&*OK6&KXO0?gL}_z4_8F2zJ}#z+wUq+^Oy zC@PsikPeb2#`DvB`Nv2?Jb{GG6V>_E+=HmRO!N*Gi#@7{r7VmTb-Ls4P6Sk=>(;+l}KI3!c6 z!G72GOhqz{X#x8xjzmnBFmu1y(O6uAq$Z_f_9{NFf^?E~lQ}yJ_cNkMty0Q$#<#+i z+>FvfT2?`=UPM6+wJ&|FJl6a4uu{Tkw7V)Tv{i80FHH8XP%R{Uh5v_=l#nSUKWBP4 zKzjI$V&ML27g^LH5i@#TYbdPSf++_-W$6jTYQae;l%AA`q9~7(!%`7(^44XF$)v^* z`Fe-JWM1LVd`R|m@B)uE2u5;|2`BE6fLkoZQ-l zIHi>n&>IfJG~Z8>YJn2~)@p)~9RbYs#3=z`5;`MUp1F(qUky-5pLj&hbmt06XZwnk zBRS!D4B^y~a*0Ca3$0Kzkm74i*Zhi4DTU4Am)PTHqE*qA5^%*_U)KQF5RHPf{!6L7 zh(Bskb-`C-n{Y5BgHagCOx^Ov?feDQ4N~Cp*7-r;wDx^Z7QQ!FBd`dm(!XYNez>b$ z8%bKSI?g;wg>6Y!Ri4VXyAAYd3Pz zQ=Er!L?$FfUAX5o*+PP?uKl90```Z`nH?BqLgAj3|49SxC~}V2b&u3EkHaD^tMbM zsmMMJ56|Spwoic0+HF{5B14B6T-sOZWb}w1h-{MQnQG&>nwqq1)+O3-e=;<5a-s?l zBLgHkmLn3dZXv2TkchTl7RrJ$&Q^XAt@6(zF{*a%Lt%r8bATt_TN?~k&+jO&E=_s1 zR*8wVs$|4SPwccD;i9AQ*QBNd)^B#Y%BcMxD)DubiKw0x5?wPobCh3!7=c?qQ;LxV zR*H{V3+N^A*ID}xtO)-U)_|KW?)QJ6%i;oe09L7+WT48(M8c{aOyc9EKUP#EXUs9z zoXo@%|Jk{cydn_5m>^p-aGRes7C}m9C&D2g70pbbuJighpjqjZm;cTe1fP3zUE65! zJ<)*C@=mhC@{-)|&CFS{oPN1l#iKS2$T65hOt8C;L6Hm?`YQ4^f1&2zZMK35>4!Vd zdE7BO0?AKcIvY4oEpgsvzv4ezz&S|RFkT<3Gg-dB-pxxpSHwL&+S&2_cP0K6k?mn}vNS$*O=76^o9XJOf`T~c$>wDg21qEr;>Y#^^(OLV1ZU_m zdnL}<0f?-P=z(bP1rE*y7G7{V(K3wIl(eRWn=c<))U3>q;-dSJy4_4R@{s-r!id+% zE>L>GC%HBn5tPmv^yTCMbdk|-v!2Ty6*X5OdR}myFP*8?GiVW;sxs3S$c>|q?hxs( zY$$-7w-m#cAY*m+EydwE{wc-lk4_#x_udeVSlW;lUMH6D{e*oxGzMtPjs${zZ_XeffAv<{##TJR89U&A|BndMUibV1{*JR6v0`@cV z5;J1M?!RiuDlI%?dS&W3G6!L9jd95iG%>BjQ}Rg$Y!t4i{!OE(;&R{kn=pGkU$SpZam1 zzd`3;YwawuQ?c9wBC=cv3xwSkJ7^-u8vpSJzps)_3Y>S8aH3MGH+-00A}Hr7)M{>&iJ!T%Mb=-lnxAo1oN2T7BK$r#e*;yow-Iw7-kfhNP_UQ31!Y=XPt`50@W zxE^c5oW7#hGF}PHI0dmpBF_NUa{z=M<>=J{cooJD>Z^Cim^LIF$esTEZQ+jur_~CgM)Wv&8&CB$e78zuSMzvIa?wS@RDaT42q+6GUktL* zCVffTkVtK3FUPaG6K@S^g$ko9S{y>k}d`k10#Vs98Ng7G%IGaT+=sMJ9e)?W) zUy_h@0!#WGYG2OaSs$qlW6YlfW6CjivMp%NW1Hj3;(+$94baTwTw;ulhTI8MTZYi*Bn=bws^B>1znD z<=`uyC8=sRD!9$=lUzC;)ZnhJcimrvwW^-+3%vwDF;2OA;Cf1sxMp4rU}Yeh`BAO^g@P{b9fZM&W7D}!&m2; zzkCWVU1z8|rW)HAq$xJO@Qt~oi_ zwz_b~@BiZE;jdX^CPVOtFM}y?4R4?@t3$QaO~KNmk!+@m{rxv=`uvJ_XZQej_+$i# zjQ_Wnk)ilsyo}u;0*Me!-UDf#=3y!J;?3T?w=|qUv23|~tML!g%bgKdB8jWVP%pHd zIcksp+mZw3KP)*)lAFcM?tX?TvSdjmmoq&X@MRUx3mpF`L_ptJ;^^odXaz_~jJ$rv zy@%%-&;t>mg#!#t-qF{Q1c)9(YG!;n{$*z$H)oQ1jY!`Gm#za2X8J=G2t)qCEQ)^_ zwZiB6lzbO%PS4zz&bJ|UW#?Y-J**Rc*EEm^xWH^cCn!*sb|h_Q?|K)|_Lb^h6SXo? z037mN$F) zjFg81NT=tD@W530-Vv4P6Vi)^LW?sB?ZI#^LYKpe6hGR%F#qjOadiBo3x9Ev1Jj)i zU*U*Ox*dZFRaq|pGS5D=Df#qISTL9mJpJDz)kjA5lydx{6x7sl0j(Rj0~p8yBtP^z zHg6c#2S@z`dcO^yeV9%p*jIIAhL{kORCf_G zw2J)l?XqU*O1exrnia7q_^Icsm+C@HBZy&klmlSHZDVHk_ zFP{mI#}p_FcwF@&9`~j(*+(n&gNJIb5nXKtHiUa&!J$}0Xoh)&`gw)Hp?5?ckce7P zxE4FaXnupxyCSdg*Lib`aBAkT@H~^CVb9y5ZoV-XEuLRu!@;TO3#M|9rj4budk&VX zZKif77v5Q4f8C}NnTO~heJF?Mp+f8&wetOS71}8)PyFZ>*C}|8cT3)`v$5=@R;y)m zw$yiV6*zXBwaL#HwP+MGbc$8MGn`x%@$C*@VzZ;2mEIeuN39zQ>xo(>g`$63Lnl3C!FA zK`*?3@%LBIUtT*6Bf@y12l3taOJB}d zbjyq-XM72~MOG($2eHtHh5t!0UVbRzjPESsnU!3xV{b8J#PTk>_md0Ui>JM;bP2Do z>`S>FB_O69VT}-l^RjE1>44t=1M?TafSpIxyrfSu>b42_Ij6abElB7n(51UmjPNVa z1;tOty92gQbpl9ri3(ab8@ITA$#QzIl zO#hBAtt$<52;DrK??=14Jo9E7%zM*c6N5|C6DLJEgh)5LKCdBN_W8211(LfiX?cA*C+!=U1>(q@p{Oj{Nl#nRc<(#qA?srSuhYq~^8@O=d z=Vkc?0JZD+9sUH&Mf||O-ZA+H$Z%pI(Yn82MEaYia{<8uzKi?=U3{An+HU~giAwQH zK29H+`oN@;N5Uu8j&{%OxTzDr zMajbX;3z1@bb2D7^-8If`*){{ImZ9!bQziXKXJNHnZgIT4I>nlkB}~26<)6^Sj%nk zbxef=YHp#k_iGE8@Hv$0)KXU~e;BH1F*|+_(YQXQ8z+Jc$bWV>r4#*ah{pNHnd%=y zw9K=$!)LEVKNZ8)oNrgfk6r_~7)K)f@6|B}u%vJhB4JU-VC3P9fjP2)`Q@bL&9BLt zidxx}Ak;b6A-`CRCv;wj zZL8ldlND>K5BFax_G#eR@AMyW-H0i6~NTc4!CK?FSHnv3lS7>7x=5Xpl z9^9`vEDq4>`lMgu8tw+ti5FXt=TKqO_<0b8t|SB=e$p_{;d%leEQ~Ac8jU1XA$YVU zYljM@G2zf=dMRA0!pLbo4iY{c&d?>b@iOly+2`NKL?zoNY5z7R+O%%|4`ZVAt$#W; z>YsKpJpYPjWc<(341jN-ssFzP8t9BPw7&t(irD{4pgC`C`X|tgb^HZ1Sq*{0fao=R z`d7SXNUye`S`!RV>OSxx}t4#_RC&xT$^^jzTaF ziMJ0vHL&L`$I!(t2zPXVA~Q73B^jUHDgk`=Wsy1WPqVYLKh1zSN&U&pR=iz7h9ZMM zdxVJbP}hnHcN>t_y)k?^A6_d|_F8&Kj5ytaQo7>#Qb;K6PZ~aC-|ouZ?zFnTK)6BU zjHnZ~@}YIYIstjxug%cEyEvE?;!(y&p*q2Me_e4|WuRdjoY1{`r(?A^m%uew;Ft>iAxrYxlK~%&!fTGN4!cGN}d^4Bsdgl0h zB=ss?kum^qnP?)R!sMF{m}ruR*~34IJ_3-uY1pj;lN-W0c4;u9<$W{1RZn8z27QnZ3{)QmeFQeB0XPkCKuK z8kE$ZEk@;y`R*RmlQxS*0B*<-A%IQ-)q7k3^85vBE&KRQKt}krWmIUmVA$|_1txgF z72E7!n_CR@8b==mx1ju=Wf(QsR0c3}elwso^zCEU^7PrDn zjw|QtS%C}@Kzf$`=3^CLr^k_?jZRAO=w`Wt#zgPTp{p6!Bz{&aV z<#A)z85>0~Q=D>!H z=30{Xq3!Vx30yH79rfQUc;jvho!dCS`+XzQ3Iky!@RPEtj#_CrjUnQ))^J2ylNv6eDAfhM;1UK2Ev#it4tg@L?T z!Cd(Si&vabZ&L5Y+(ZWl5`)VQA{LeU22Yp9U<`*>C362vVzKsJvHYzM+ay4zbH(jC2cj3mx^A+@?p=x^JUs3ws`oa5voFVC&^F%;G=)-W(B{%F}Aj`!YS zo;B87a5ud~<)`#$b%WT=4!cAy46N>08 zM?f?U3-mrLuKjyh7vB? z1>J0KB-rP*OPMkbDS8+t3Uys1JpdZ`lj(pb#CjC1=$zuj`XnSCAiEFo7uA!qGO1ZW z*DS31?Sk1%+Uz=S0=IU>|Dx-ZfZI%y!fp6qXu4B>O5+<4R!T* z&FWOWAiDjXpdnqOei-A2LO+#>P!1t!^!j4$CM_~8buunQWExVD1KX6CptEQ*^XWIk zA*k~u8g|Hb4T@{mnD1|?K{GaS_+pO=u_`K3sHzn@nbb@}I+3S?c-Vtmr-K-H%s&a$ za9}Ajw$*#a;ePgKmq7&7^c8@E7#WCN;nyEFQ|ciKiXm0KNK55WO8?3dK`_|Knp_=8 zBLr^uoK{xwR(G^joKyude_Vv#sz)o%@Q0 zZFTY1ox|EY)U*a(N)_ZoLIvcxZ6hs8Q&w1}*%a4%UrcY_By*Ah2lWu4#6Jdx)%@8cUHe1gyi3Qxcp3 zGDx#Oe>9xR-%)>gkmEFcr5H$d-c~3xseH*vS?n}&qpcs{qRuZnq&=ar*Fd&Zrb4&a2om^&0df&W z@N5@p3%heG+mu)1ug1p)FCz9AyyqRT9V6)301r0wpB}8?#I%U)^FmA~5B>fO1CzdE zo=tgBsqE((90_d_q3d3P`uj?8o9PX*OUun7!<^Y&zK!fyzXm$x2)@!4^FS|%X@748 zv#~8GlqE^yv3B=gordl~A-pjph-R>_pMGo-#uWHv*4r-xx8YUGW)ytoOgtVgCLkjC zBpHOe+W617d5#g`KZFZ4i2RgHOUt>T*V?BQiz|)B4{%{CX8xNC>t2{X;Wy8H{y1jc zxyXtzHWBKy0*IL}Soql5B7)=8fH<7n(5P6;W|#)n0~+N>zaixbA%q-Yo<#1hSg(0W z|9_3xa=Fyw?&Jq)HlgwV^LG2*!uZ4`NF+&DFCypxS4W0 zevsVsjT^RKbfz^{Ha@H!p9R~vKZ1E{#R%A z2n<7^-{RAml>=$x2pjh8`tsw831M~jap)6@2Y$~qqz|~rY)B_KR5lk6o0r|w)fcum zRQC&oVvsWe?3xA3pQif?{@=f#Kp?@l*i`_BfTHU=%YMC$7f2vdPZNx*P{EouB)TGO zRh7lSr)BswXhCNGhe69u{11b68|_|q#^YZG?dm@b+DLu)51Np#kCKeE8v{!M{9;f*9b#;r8$`a)Gee|N^M ztd3Cjb`dkaMy)`JRrO5$^R}}+TY7{LU(~x!GpcEcys=e zEjPq9Z*rVS#hge0*^*~VmhOy8i8xy!sD7boi@Va!CngcYdCr@+%H(~Q9iNDd{w}nu z5oxinZmOk8UTk;XHq?>Eg7mjUNhg;VkW3A5EnS z*vGez@y})KMFRBW{6@_-egV*=zoX8;6>q5FtB%3F{6v>%H+^Ovb349)+ZJ+Zwxj^f z!Dey6jDJCs=->mdI@2#{zp+F*S{1}IdJJ$GdclAzQQr9?Qz@<(vxxv({uNFkVLjEB zNq~Au$vJTP@w#|rt+-GO`7`2wdp@aNErXT!aqa^R<+F5syly1e`7Z`Q1hkv84%sWsmuBpNiLv!k?YWzF>sQ{|diV7Si#cg= z1;VHFo-L$|DaOGg<;6@RlpABUGlm}8?aC1??M6Ok^#Z7rMIkzwE9W&eUsiRdv3R-( zBvbUgR*qCBN9m5pvdauQZq06idcr}8ZoEM{9Nt;oUv|C6=VOU=>CK7|SR=1-_s*nK ztI6)1iU47;n*1p&-bnry7E)*bE-Ykg8^)aeo3OxxN?teK+S)QkEFOFQEvobJxMEs| z>$uAEocZ%`S@5x7zp>**z?ypwAT6*t$}c-Wr$5$3J|5LyJ9n8iH@&X=GU=YL#k^s9 zI)2h1_1dwr8llG#RAXo$G~_~v-(DO~c(kG?R>zxrb zqB?ZD2#CTIFf4TDzWjYU)2jRkG`_G*tNfFK9cCgweSS}?om3(wb$teKMDHcJ%wQU> zL`oZI-Nh0OJ??rIfo=GOs;-gF5&hNkP-l4PnosNctSi-txuIN zE1e&SO~8^`p46o+@5EWi*^J$}yF&c%pi{SzmT6iqFPW&ADhp(<)39d zI_7g=bUL9Pxu3I;o;J)5{^PRpfIJ4+fufeJ+)ZB(qH17G&DM6S^-9Mwv{IwfZpV6t z+O9T+d$l45g11tm@aFvJwf0^@H{L6?IS`Y;(^Ik@?81>zN8;_?d9kqy;i7v|*A)Rv zClDw|R?OYx9kP`i@pl6_(7OSN&%6O0^i%t<4tFoBKGwv1{LoFQQ;)Q;;vZ(0l$4;V zIqQCowH7cF#n`LmN+--1H1Up&;>g@3veio+F9bw`#eYx-$5k+PSw6^%q%1pyn<7|+ zyMG@dkD*D0jk5FqEFBGJvJ-0L_4+{-F9tPe@N*#SoOrxD21gc8DoY%{Qn`;0e`C=Ug_)`DAjjgZ_`7;AQGs*6 z#YWh<*s|5pN;VApGa*BgCVhXh&oNnjxZJ0hkON(+)a%s#rlrQf36I52EO*G1_WLJ! zl)c4*_)s*;O0}jv;>b=bdSTzLc*mU$nnb6(H^I83w1zspvR1RZSIrcG2Xh6!hb2JU z;TyQlVf@s!qbqX#X#&N{ih&kfg9kf;oQee-7fIOdGna8#637orD4ey!O8>iriQ6BG zJ0diI#|Ch*5AyCtSXCVC#?^qodG>r16>VjdeY}BGMca5`GjtOv5{`U#RWuQMKEJQo z$s(ei{E^ZN8ZQz``}dOC@fqXyjoQ`6+DTQmrfIUwAAQ3n5MV65-p~y`BCk<3*j>?) zS2Xr~1|e=!@FQQQWHK^%sF0=#3xNP{_s7RZ!`d9&?FtqhPqm%jShfQCd@Fr!ZUKpo zpOX6~_g#%!mOmy>kE^yxa)vqRwBcT<)Zvb9-2ox0whTas3ec^%ZQGdGP%BAQ9E9}| z%A#mOM{wvERI8=V2pLG>GmwsCCW*na0Icu6FGyK6fu~|wPJB}ZOT{;#0*+)o1ZCcc z;|6F6NQ*Xg!ne6`b#$B0tJTb3r(52emnJdzG^$_S z+pf!&W8aNa@ZKXF;1U1)W4A)RnQzd<7VTj+6f6zj8jlE#2u4Eag zH_a8-xHqYfuR<5mFkt@g2*?~CMh z*q;MOwQPEd1kooZ6#{_UiOw`fUPpfwh%GY@9P?B}7mVyn-0@Jh!yel;fvQa?ZxsE3 z!g!7VnGe+De2ze>8;u_ULe&u0DI^1YdUw}Aq{JG5uKq6sF`-%Gr~u1f3wst5o|355 zzC_LTy;K8rRJHc^Z;v`46LZ$%)SF{VBKcArvT`peWMXKsH)54paVac`QksqVFh0}>0-1O)c z+C}6(a{Nme2nboI2(5nyx8t=6UBCi{qXiB?|5}FN?5W7j@o!n_8vr&S9}}fN?NZBK zc4|*P5mlsoeL%Cv-MKt}lgo_E6h3_?EhzG)_w7>We8E09}c3r)>UL%aA&Ab z`u|*p2;KbEUX`dw#-lUa3>2LEzhkNfjSGJLg{fkZ{J&zV`tz@Kg|rhl^ccWtNm}_Z+@(E$F%eeM2SWGm-PQ0=Ew{Vl*7ZnZ~rH)%B2_!$T=EL-NLv{^WP)0Y8(*g z`WzBUd;cyPe&WyrFSon*Jh4USVP+x0-GkqF>ahNqMtTDFMxYR_%_+RCd?LGh%X7Te zQk%;cckw^2OZH?t`3=n-@^_iF*d0x(vJ>pLxsTbvemV^)IEd71^mhVZu1)&CL$gG` zQxk`eMW85hs*(iY4{o&WKX)EgO7ykw^s973Ktw>M-zl~uT4LaaU448AS%*n2J8~Ge z8mgjLKX)F_Cjp49+nF(J332Bgye$Ue!{gtVt@EbCCHUD2 zc9lWFP%8cTJ$HytAeq?iZLk*TO@*k4(XaX2dN4t8Zgh18@mG>I$y7s@tubu-0DOVU7~aZA z1H{>;Kgud22MsY&nw`_xY*la?x`2LcDB(mEF0AJe830(|d~r>f_^BOvH#)~IGYKX#xO z1Fafbvif9se%(p0UCx4TQoOq5$Z#&{`Lk>dOWO(izB?Dup76@20JmX z;BixQx2o}~<~zJ3{2NUb&uv(C+hkv{DSC76t4uB$y^f9E!-UL@r3ElLz8Edpg!A?O z8lZZ$&JnM&N5a^8$K|ex>in#pDtmOnmYJ3AT{kQHJ}tw$a!eMx z0gR~K8w4b(&`kgvT?-Ly%)5pjI4HSR)p~qWQxU7C)yO@ac%R<;U=wlHPY2X9ZW3g7 zQ}gDz13nwY!@z%jMC^gLJRK~Tv)A}owyv;{QR?JzfPa2K96xK#W26=9d^^m$>T*kw zi|*`~0C;F+T=c<|74Qdg*F8ss>QiO2Q)P=MhrH=U=XvNn1YgZI6Z=h4JHBfcIm8LE%u1G&sES#BOycAia>Z+x%<-?c-+A5hIB=UvCz%KS9O z8l+&3(74lFNXdI=+Fv-x1K>xZJJyS{V0|XpAg90?LE!aacwCTd$(`dO9Zo%6H~dVF z#?CYZ3td)%a~<}&vgfNOTME);YdDc`Pq<6_eX?W}aZ<@)|Ij_g4fs@Xtb2EMB7SV= zZ8kc-1`oQ1?eOMtmkIJ5MI&(R^%e;5I1&n8gku( z`)k=*IL9Ss9Xg2FnpS1NbKIF&Nu*ov{yaTuvelpV9b>Ut==OqW>4z=H>XTN_$C^p$ z!z-~~CUh>(o~HTn;QESrMcBIsX081e+&39dV7J-U@xl0!ydAj83$V()H=X@^>(^oj zZ)+DLWmMk9J9F0eJ;&B7P})?upH*Y-QIfDz^$M*Rsh!hp3pV z9JD7R6~{H#<%T}c@K!J{2B_Cj$`2;Xo$4wbP(8a)iHX@)4Wn%@9^Gn`*Hnqe2Ko5S zDIcz(CL-EC*U-Gz&t&!|&jG4Wex>XL6n{KK15O0brl@8c+^x6fFE*Q{gZ=60dxo6e zA7A>zP{&{Z-9AmlD zmJ>mk6oP3`ahW|Zq;H>aJi^~}eJO;~u%X6cXy2FxYmx$t#lrB%qs4fOFk_k{m;`e$ zV-mZgZNbI2^eQNjuVN@-euZLvEu&$UtdUMI-1Lem!;%q?IlavItvG|iL9F~!`?r(0 zTT=YV6=l4h@(I0;I0ou+N*0Ibh$2USksXizm$VVF)SFctbQ?Qr&89wi<&kg3gX*Js zq)YrZ+?kuIt*!4qA=CE77&(bvwBY+j$bUR2M)xRfzA8tw<{;bTL)#9;WpP!FXz`&g zvlWeATE*y;DZM|nVdcI&P-9`oDxiM9%cINpv}ggBIkre6ik)IJ zdZ-P(?fB_*NGXVHIz@CT_1)Z4+gQ372V+xR4lplDl}JRB_J9A%AID3y6E zCA|SDvF>SFmC~Irs`1&ABRL_J7*kRtw6Y{~07@Ex34oHu4WOhEngS?k!T?HIG=P#8 zSdiT2Zp*`5loCqCw;-j`U5KD0noH$Z z=R&wC9fSarWpcf1sTQ2r!byk7cE^tdQ!p_)Ls?g=(OOl*of)8$HaLXv_aLdewi_zLd zZ$r`CtGJ6v+6E8%TzFu$hwy0TOSGtSRoV}xOFD#QdLBV{iU9Q3QU{=z64il*{b;q> z(|PEP#q3!_vka!aBEQ>z%|jW5MRs#K>Wh4?7?#g`&PY~nmMNq%%aigjUH41qIh30j zMV&|6z8pe|bQiimEF8Uiz2KMcvLFCFIGy}|E^=3^C zrbiO^%m^8)XHwDvGIPirvXZdv=N0mC;ltV#hK|A0K3B#_fMBm{an{f48qwY-Xg#=H zs=Uk=_S2px0ETUU_&R}pgynJw5c`ZY6^J_MCJ!6n*tq3N#at z0xi@Re&>XZl$Vddi&j0+3Q7PaIoEKN%z2Td)l%+ez2#6SB|eQYH=E?;`oaZFBVU*G z_eKzGiJ!%g1_C^mFlY=)L+8joZ7b@p;iLi4PY|6QaDW#d?~6SMXu+(^9a*RjnkOd8 zDje>emlIpY$Z`K)=}+Y`K>Bl5;xql}Y##XDYvcF?2pwp!RaEo&#RwZkE=0-}i7Ohy0uC->wl$jP>*+PWx-%H4TsI#8K){y zIt!70p)`{v=Cf|wE^ScEa9$VBhBnMm{}8ST$|1=scOOQD^ddp%N9lQ&h7>M&RYDPnr?qnCDAH4EJsBu& zb=y^XavgE0ingwlL)!?t^=k3e4)t0k(!7Gl0dF zH~1z}&Xo|`%QQW?&Ls$ZZpU0DP{Lx~8Sm zn73(fEg{#XV5!N<+XkW&X94QlzS=?6CjGe%g1SliZ?ILoZ?1vB85D+$ibJIll(2#S z$DK3|9Vjkc4j}B5lYRJq-AQYit)iH2;&`uukDGB3{QzLjg9I&ie1T|n0{Y^77f9~5 z5nGdXKQh~WY;1NL-i7TxUS>=A3MVEN8eCCH)Xmo&a7M-3unQ~p>srYs^tv&lg_uAh zfFfvr-+P@OV(MP>t%;ooK)Gi05gF@kH&ppf_5l03vd*C+$v;A*O%1x6`YE2tz~;1F zedXKnwHiTlRBhYWipW$j&PCa4y`!AB;M7eezQn0!ox3Fz?0Q};l@oS+N$y2TdG{)&5v!Ax}-KMAQdvm|Y5&CBo4zd8dIb&k{#(A3L}WdGgM( z$B~VjSeLD^PaD__4MubM56vcP0sMp9n9(?%ULh&Jd$>!_UYe?O%O4gb`J8ZCsuk1S zx9MdTBG#4~TNX`uq%3Spu|E2jOi41=Bvl*fAu znQ!HmBp;YWIG$|54Zx2oax_PyIh`%y?H-pV8*!yN9n6x=OW}}8xC~U|86`c~bR%E; z!CBIEJfq7s(1#!tB)H@{HkJ6LHmA}&fs_~rl}8IsMR};3hyx`aB6uN!Ap5#kv72Qe^;YSI%hkFU!wiKo%jXJjGdim zxrbJBhPNH~Eq&7P&|&$(qJn18g|_M-4*d)2A#kpHzX`~%-KQpS2)vHSC(D1K$G;P7+kB?kPO^X-@+2mSJ&X; z-kmmItRTer()(w#=(8#)POW^+)ej~ORf#M>O5w^X2*%e2QNkj-N7)4e5}mrY_BHea z5?S-4S9kVBtd${`oDn3#9LO#c5rB-CHu0On@N(?N@G=jLoHh|$xeLYg zl9})6Rr8GKCnITA43|Z1Vn)N)rRG8C--+cpR@!ILI};e7RYKNS3ZkS2a~N7_^}>kt zrH4|A776cWH7@`uR2YE0n?%Aos)PZ5UPQ(|>Lj#UL3=*EWr=1?JR%gBD7kO-OY7pB z2nJ?{OKST;r@kL~XIizl6p#omi^bLK&GiRQ2aCFxm`P0m9X6qighkC>aAi(+n9QF9 z0WJaX=wy%Xu!2MY$?dma5R&@(Re9YpjlLxG^lA#Crd4^g7}V;LmQ{N{>x#Q$WI-i6 znAB!;#kpYuNF*uX62@Y=u&9p-qA=}YGnicM-&}8mpc1>|I0OPHoVs;K08b6Y=ToCU z60`)zd7x3SKTe_QrEM<-TlWKV1s~JYCaXOo$nPOPP8M;6H;ii8BI|AGCr@2<6T{R7 z-^;zANAnVh4G|{e6pyHBRy!?*=?_WEthb+aW}2pzldHNTS~h2!N_Bobw~W2nFpM(E=sLOsF9T{B@}zu_O%mhhi+}F0M(Svw7;v!!P>^r6a|mDVFbLc}`0NRtdmp^x6IfR}z^6^b zF3_egF@x>Obe;Ca);S$k!gss71ds}vj*q|otI2_zGlVmS zGHUbGidxFkMeXOul~=MW+8g7H)&~wWWWfwnw71J$+YezBsd2CBoJ zDFC11LFLn|cV_iP!{D>{QvP%1%L>;pF;E>W)Xz(->49V{F$;q8bN2vAcE%(XNmIIV zS_~;oNlUu(&+oxHS+H4^bVQ~kGz=*VI=~0(etvLwU5F=?93tS7HeI=2GPtoyv7wxkrbRTMBFZwW?P3)XMdoCH&DKZB3~ zds9mH4bqWfa1L-WKv^+NsT1sm1r5%kLRWV^qRY>K`h{ksDo8tcrvYr{Ml6*dVN4N&_SqlL7&lz!YbiW59CsBh0;2>%1=J zcS{h8{coBS+I0P94{ay`ewm|;UxQQH_VZsE-Jar2nKM?fW_EyFz2L6CI_3vnT|{(K ze5)F*4zL_*f#17@3L4wi{%#8vMJMBOFazb-8qyUSj$`XE+t~p=v=6zq2HVRkj-ejI zNZNqSc2v&5<)IMqGW|BT}Qf!$vC3(vJ>$;;}D55IkJQSo$a zazPH8)Ak0?+xZJrY$Cd6PO^Y*l?{Vh-qF|ln`dRZAJ5$2@ws0k5gHH-y*O51T!bka z<5$FJ+W7`#lu6_sy-G6D)}CxiGzGF8u3kHL7^+Rd;>B}8# z`11=X?q?ex#Z~mPjn6mrl$&G#UjR5P6o!fJOL*i<;iT>EHw@d9l!j~5udF;z{ny48 zk-Ea&Vb2}c13r0+yBqPu4KY^dt9+50D2LPC!4%Kxa{~kit3)>s!Z45cWQ6GtAe>I% z-(e8ztW&`#XZn6fm+#z3Ss_I)ATme0fAj!D@*?nm4ao;_OM$BYH6*9oICrs#Zm{dy z6n%n@?i*FJRgNAwy(ATLg`Yd|oB(5TIx)A?dEg@pJ)1Q@zUXmO_aEI;Sd>GIwrs;N zlgsSReL^6?BKtNlK1iUv%qS+81>+g%qh3!X`(!}-K7^vGV>{iZ39Nzm8wj=4FPB2U zQ;9pEjLSV|_^w67M?`F#vtiz97c8m5CuB8TNh2tzY=MiC>)rBG3xXq`tV0(1Sz`tN z%FkpckVsgm5XGz7#LaItE}Jd|Qbn8|$MTsWXA}T&xFUrt&aewhO_vjAV|D0Ocy*7H zbBpQ;IZ!eq@94Tl>HA&B(8 zB5x$MXq22$wghZhRU!nB_FL2p&svkwKhRiK#VyWL0>_A!=amfaOJ0dW&H2?F5mp2=weP zLCvV(7dUPrT=NP~la<(^rh>fZY)r?q{!6T4NJM*c?6(yI!{5C@WDRw}M-1@EH`gEe z2>n;bL6oH$a+&jFY%V^b~gaVk95Beh#NkkbR`ejFD52_?+eLJsP)v z#6gI8V)VeqFx(9Ch)twQb}8bK8wbY4m;@FtzBRaP#`J;p8a7yqYm7jd6jwz4hH_$Q3^Wy2uNSP1-TMkzsDAr^UudH~LEMAXUth;eHM|g%iiIdu>}zlG>%# zf|e9;zkSO#6`f7#axZ|)H9#<+S#KLF7^h!jBak&iKsm7Sh(c+dUYgmsv zcTmX%`Fc_*gxi)CdL_Qna>O6h>cI-j-xwoNZd<5Zmit}-oXCt0sr>knK83NkS2P$| z?c>cq>1foE&-0-l<7})B=+)?aG zcpN}d?L+mKl$3!(Q%f$QSelOYb_+@-n48RY?Gz)waX~J{@k^~?awe_b;n431Jry@y zHda`5dhiKm=72Lz9WG=|DlShlR+$9oP7ct{!QB++YzXbuBDSh?I+9JF@VP9fET^n$ zMW{p2i-PMX$nkiFm^B=rL9v4BtH8BJE~ii}Cxj!dhiN)wHD0wiNrTVqH!GuIH!Pzw zN^YRl?t`iELl(o{wSyhKJmb)_32rG})cCu|zXZe*Xk=kZrI9ctsiBXl=rUIW_0`X% zLWYd1Y;iOKm!b`V7jj#s88-l1A&?zB_HbHPiyV_}S6$slgyF01s*4kX6$2;sg)Jqf?dMGX`f|T04`KQYPU| zJlf6ya<;8YKY@pdI{~_Lv&>qC@=FYKk_@dQ16*H{%peLDVz7Ib> zI2^Aw+d0eKa+GiFOa#S&Grr$IVi>x8^OBTF#A*(K*gZxF#F>48#2dk5Hs+YCfW-3+ z!M~BL!HF>t9AEqA0jA8AT29osAW5-WgF>SlweF+eK)EDw;1O7KDNx`M17#zg5=hB< z{sTXN`zi)x90O(<5&;Kh(N~Y=O<+*`mp(s>rxdKAm9I7Lj9>Y#kit$1_2Kks2pL0| zx2+LN`OAKxSEpFDI}sI2K!fDN#xclILW4{6m+6!5cT3*OnL~Oy^BE{Py^#Q<1}muK z+r_krMdNm?V5&1?ttAFXs`b5btlVr;YT|t$ILEPnvs-h@(pG`SiFBVdOm%AgEZMoi z`Qx7kruzCOZc)5fbbYOPJ-QL^ud0-ypQG{5&(U~3+RpmK^$8h8GBuYk<5IwH*peSt zllDNM)cy{HI3SynuaYm(zRgoCOqE$)EMJ4bCZMC7J)u@0bU_R}*9jO5v{3nqIfJQ8 zBg;abH?BLZIm=L8f7r-ugRz_%;b*}%a*@_|HP~^vBDgXJv}?!o;wj+l0&U4pZdB*a z#>snS4Hxz>q$q8Ta6rBck#wc@`u0*FHB#>3F1@3q3ab&(+`@!BnM2Z&^;Xv645ETg zrxFX`XG%?3TNi3miZpcts+LNks`^5;2C&ig0$|=`959sF*O@;oQ{nI!=GAP~yZ$EO zb-#Q=-tEsIWt>@_s(r3)hsaoTR)BoJ4yc$$|qAn_UCpr9$ zI21UFr6IsW!`|A-c7igrMFq zi}C|(_*D2r7GyUKf3A%R1%T*jG9#j{i{td;CMg*&T;rwXK8*RuBKPrb8blVowK5X- zDEIN%-8@~s2g}?5LPn?eRmD>BEZfexs9Z(gYa0Gecs9|MZASDIPqP-jzS`qWzg!Xw za9g9ib>W`*s6=luot`ZS)ESp4$d5{fAXApy5dP!qbU<1)eURMo4&GfC?*tO(;1Q7| z<`7NxSyt@?N_5^?I4i)u!}SdTv9``guDbT)*Plc>&9msy2~ss3{uqLaEkslU%&)3!*HWaSgbi!RzNDq)m?i8p#{RexMts4O)<7 z2ZZ4dpQNyukDfH<8$P^zt)iu@JpNau38;IggjKX+vHCC?Y6Ar}I|G9LSQsfN#bwV~ z#|Hkqw%i5~rO#D5x{W@IU5g8cmEU?o4RPc+>hR-9(?y4gaDBY&1{D@s$F~*0mxY8$ z+rgDa+z%=QoX@aSY@iDz04AfgN82d`u%$o&Y$*w{B;2a;-!4&0*neHhd^=YzLTq`H z*_@d({rz4EaC|T155;7)IeXh9B~lkGIQ=2*Rbv?Zk}RVT6aFgX{YC#D?r)|2p6tFl(1Vtq~sKUAlE$ z>tICQw5KjRxJn$d8{f|&L~IcfY>HHy$*B+Z6VJqB5ObB~CUogEnTV*P-L4p{@Ax~7 zKjv@fLPJl*YBj@{^FliZWhF@JrS9=+82en;Gi>LM=`pQh()h1&>O>cuBa z+J@3f&%&_`8m!)B#W*^-1OSeD5e7ei;i#b74!^f>`2l-};1W_f_2qmNNJGhUf-x8K z6npunsP`9YN!i8YvP(X2lMrp>Y+m{(Ci`cTdXgx^G}KUS3i@zQa996fEn1TlKiVx% z%=DUgcahD{m?Ge0Hm(H~r2K@W4o?(xx z8s{5$JK5f5Qfg%u-uU&ZP8e`ek9thzn~Gw`b{MBu>=SUjVwCB7_BD|!D!&(x*RRzM zeaKhdinLJTb_|zr))4RY=ipid4=1(vT+g?)B#r(*yc%jkgab+dFB6$0W5@ofsdVwd zpuH^kFLGmm_qk7BPA*8t#{NP!YsOi(p-DlHXR^3_x#zQvK@wC99VlCD%r>^K=$!{4 z*~onA|55pI0o}{BqO+GYrqu^l#w9{v@^+?mSL03_b9b5&Zn{LYP^y{q6Is@GH6_## zoLAI>=KCo};%@n&F1+VHaz8|extO=^RQE`6UP;+0zn%vIPLS11bv=>3Wy6%2Kq&6r zI<59T$+v9{rId%k z*RUmndomcUzR?Rp^~0R@JG-{ST|2e5J}@)l4OEl)N8@kD7q;Us1iwbTVEbXxbdX-q zDzMBPUp3g#7naM${hI5gudrQoqn;1?)<#GzXVa2(FEHFFqiav*ccp{U;FhrZicMpi zt>dMYLD=r&z-+||GO!wB8J-Y< zN%{B#BfUW5;D)k4nS84OFFt^RQus6NAG>Ck|1%5( zef3+1sd+_M6q!OQf!hjZ;l8K${MimWm!D!d9Sg2Rb$EnMDYG9ZeTg z;c~4v!&R+$eN*w(?3sEhw*?3wbUGHqe5M{?B!A3(sXgkVpr4|>l_SkpV$8{zuLa@G zjJ*mzVGW)Jc-@ZFU0^cSByUBuU}<11k0oQlULB~((j%aO3H~)zqs!7TKG0B2vleTW zed0Pku!M$;0xYN%Va2L6c=J=K4-9|w+6mp+Uptk9!FuW3`po#@>c+l&!8bl}9icw< zk)cX2LAT;s+#KG5cM5St zBzl337&D?36T^l0I>}CiGPmP$z%twtT|bghp}^^;Zvb-`1B(eZFMIn42@7kCp3Grv zI0^Hje~hlkvuk1r)7kwS;06cD7%vy1Jb`3kr{FE*iR)lB(J#&(xY|vEC%od|L@x&l(kl?S7Y5<$1Vjz~v z?+MqHoC$aWp%+y5g3}D)&!(gqLlcjzsck#$8hu3~mNm-u5y(Rft5u2rb*v9VRZR&K zxC)jUCKw16u|5bDl^8x;S)9ileM8+F!E%hwOZOStq7_yV6dwJY=j}E+;*}f>W?eEd zlc^&5g*&<%-pCL*Q7%MltR>dm))-kJW9cgv+yoOFBQ&^)NFTU~N;D5v8hh?8rV$fj zwrT3QL;&G!Tse0fiaG-UgRChZU@wEHF8MmuNG+XOqKgw-A2-TDYGbC+zw?0OcJ(8~ zPIzs2iAt-5&L4CK7vYD#zL~>U0PK~VE}7=YdbD55^gh=6 z)zDTmsGToMlD&+F?u3wDlz(4}lG)a;Epodjg$hxX!Hm>kfx{2jf0l9&f09n6JhCgt zo4gbiaLyTc;2D{~8P#oFxlyJG83tW)|D;#n^rt|+UVL>(5Tjx?n1FO^!Y4t^N4QuDqeVZ9Wy{tOqaE9o5fI@3~#sX6s)2PiSlSdi^I<;a zS{%(POw~q=ueF?{A>JgxDAei51#?(_LbNT&+sU+>mw=Ec)77k3`?RB(z?EEw$O-Io_$YHlGeqU%&l6%jZA;&fk z+qoI~G@FU|`Cb);5bTY6UdJDyEq(E#u}CG+hHxoQb!|d^LcGAAApC;Yi8_<_VZd36 z?F|Tp6P?YKXBl?$_j_izF+P26L7{1#>1@bIw&neMhhfH5A)fzZ=??UdeiI$e(* zffPt|tU!o>sD!$@)9BcXlAjC=)pU%KY83PEhXRcHgB_+mx9lQ6{ zwVO{RUOZiL7y`Y7`YAUVkaU+5A~4QrpIOct{II;x$UW68Z+_(__8@MZAlyJ?`wA^2 zf$g}-hc8nV4kpruZGP~#C2bJd*+sEo4l1wX3Qb_w#bN|gD-;nGeys>Ng6HpivOI33}bT(8kQXPV8e5Fu^|-Iph^qr1xx)s_|4THhBAarkEBN z`J)|OtgTrBEb3<5fZOf`OwtK@SHa=0lN*9Qqk_OceGgq>d=bTl;8e|gifA-gvk@c< z|3A9kF}SjJ``3=samTi8+a24sZ6_T&9kXNGwrwXJvtxUm_3XXRe(V39bH3bF^UJEW zYSxPx%z%n^~M>Z^KlTIFtKY}1=e!YjHGh-PhmR+TZ zwdDxH3c9H95|M(c3W97A&8ds!GiZ|QX$0^2$q0&0Mep4p4-+7=6E9tQJl>!m=5BSFYR863^ zfrt)u1VyWQXo$>>iB8n{54Tr+SjXR~G(2ZA)BNBa#Fd691U}?Ny%f>gthzKB(B!9sb(i*Pw{5VY5h!8lc)uPt6mVm&{`yXIGpGpCk51owh=ei%&g1m*uTfs+JM`}ILlia8jys=~urs&D6SW_7E zP;jOMT`{m&>&qsdFY%FC=$Vml$pB;v+GyOUkuH>{XWWYD($1 zLr)mW>+XIAEO`B^pZ$}Cj&}rIRXJUvAB&DPFfu9A3TB)a*FXE`_3b3q%N?n6XwzC8 z%5Xi4w& zo_%%HvTUbbTMoP9hx!C1MA_o}jaYPi9|lW6weF_lqe6Y6ZPb@!2$gCc<1E^}1!D?b zl-4o-yi*aBgIj8PIKnh(^VHfVRwQJ>P{bjplkI(elU;(T4ms}hLCcIO2)c@4p*#Nl zizwyBpN#4fIpN_4t&_J){JyLFb|35Fs`0@4FY;yl>Y|fFD*|{yImvBFsws36ND&tv zUCgT&SFT6SyYr)P@BVZvd6(|Br21}kJt$M#+L0%k;QDTfp^+zT;Wta$+Q)b%PR@y% zI}{i0Dn0yV`wiBqEQbs`Qng|V9a8xb`_Aki?5w9auPo}@BJz{pbiiF|f3ca|`@L`& zV8V)B&^o%39q;D;+QKDu{5^*iOC@4Z_ttLcK3S`g!^h^l zaF>CHL)!6V>gw`Mm$(aGoA<%pn)jjSeY?_&BUF@(=V&XG{O2ks2zYhv;3#?BdT~l= zJ^1OvB)t0=?y*~l!hX6LSmRaN#$EL( zBua%VypbE@lGxv-7J>wIfb*k8BE$x#k-kSSf_+9RV`Ur__k_&EevT>OIBZbT(w-?i zblS+s#I_C7WH9e8&dbEE4bEl|$sx8rKjvA5lj9>Wrn-L(C#KqHaksvn9ZN^FzGh@a zF~=!!VGT!G>S}34u^F|!yanoD!%!DDrnw(4geC-I1S(ev*Ga&f3eQ_ChW}8>o&C2i z%+U44-NiIBIGTenb6xLGHf_+JG!nBqzu$9)K6-5?wv5YL6e0^@L(ibM|IlLBaV~5V zAIsc=kPJ9ILKB_Q16zLaRz=)x@g?;5%tOxa`4u~H+%_P(Zv6*c(Xoes=o+Dd3BO-O z?mx#nQ6h}6;wz%r*uV|H#%2hFR$Cyn+WrHrb^nFdon_MRP?A3-V@Lyj9FBvG^W}pk zb~vuukKZdwaD0E}+pU%XT47j}X7oo5@)J6l>h=FEGQ0h&$XsEFXlju$MF(mT?L7pW z#u!5sD+b{ZtvT5-G;}$CQ#Hcak)M zE|G&tSKTq}Sc+`sczUyZUf^yM?B^(JjG7H)T^Zqo_3)5VLtuRmM1-C=gXeNJYR%BV zuyXrb9ad((qLro@o$=n3s%4PkVhwpWlHRj{l0%Bk@9dbS^RbqZ%(j!k;!iYJvw2UC zkzF*DHH7*$1g+J`QLbz6AX-OYrY&oleBP!wguL!yU{==GBfAiT%(ZN4Jw)vaw+TsKvRO1%Ggq7?lenn zVqp}xBc{W^lH5+?Vm|FJYOd80)eP?8@KzwK-sY-%YOV+Od(byfsBWau6t%nP8)sDX zr)?pBn&aYS(x9#Hv4nmrIFsSSJ1fN3y8yCntwEJrdESR`aXI@E@i1h3a0!#6_)n{s z(g=b>_<+~WOLSil{)E%tbXG*qzpIFO(((Gb$1V4{7W*wdxz>Z<-+?*;6!^QLAY63S zJ;eHh+2QocqZ_5Rtl5{Z`9LA!|5PdPqC6QB*YrEsuKy`PU;k`i1RTw<@`E}|Dr~_09Ze1fG{SNxlV-xCwG@X5;JyEOL3JY;8dhXeN14SOn z6GHq}`s0RV2dGqB{J)e6{iqb)B_wR+6*x@UoCi^)p6tFtQLeS?&G2_rHk>EuJ%gD% zL2kUK%TGTxPws3)!`Wa61!T-fJa-gsK4BHvx__n@+vO+zXXcAcV>#^;QKX@A!Iy+B7lnafsq74eplk^Q7K?fQ{nPfuG2oHM@otlP2 zoqMw1k=+9NVEp(7=^Udt=}dCs{&x+UcKD?OOR#uKf^y8`OpV9nGSDur%Vp?~;%qoP z=wF9gk?FZhkGIqkiRgy_{&S#F!6WH?zCnF&XN<7+Qy^~P{|c*p|AJM;F$7Jk=(6de zthw;lvO^Z&s3>F=L%%`Z9Y;7@N-E*agKewb4wT8j6W+cFBzibsgkNa&{r$f`P<=}K zU85L-mg`kp628+Eji(Sg=ox?u1Hk!PBk{?TP<$bQBb>i6DeICS?u|NM#fu$yfv2)$ z?0+w@Ns(yD5dzMMTIS0q17GO_+uZxUBk`MYgJNeL)Sy2t5H=ZuU+W@7F#lSzSjELa zOG9G&o4q}c(uj^$a@l_!V>-CSGa#)3hj*yWxDIptn^vbAFb?ax?18i@p#`K>)XJ<3 zN>KMyOA&at#+(?H*Rb8BSReI}_mqGPa5(C?AJHL-5H-RaQjHlX@IWHW# zf=fxCf)Vr9725Hu-U_Q>1T~NTEC~GCCb&)u+UTZ~5u3|i9FEb|?V_5La`n$P>wT3@ zKGn`1%jNn=Nh(K{Ka?&#qeM zb^fKf#eSDyxOQwp81-5?tIqhg-I(tZ$8BcDE7J)hlG0N<&gfOc@+X?hf989s{r4JR zKftSO4`g!SWr+J~>CB7;iwWD!C_`q{C06{!NWc%wksLIE_#`9&x41#=(14gTNa2fg z;z*AN`JUpVZ8^n+4-~+de?sxM^-uuNpIeF@HahbVNSbgVq4)E}^z)(J{ww3_9i)Ii z*O$-vcT7hZUAhNPNjuS;K;KQidxOnQz6&AS12Zj&P&7uabhp!SH!$8B%j#M89M4l_ zbh~0*NzTV?GVj>GqXVXLhOu>ou=N+A{eFNBb=TwX)?4X<`DDE-gsdvUCbVezhz|40 zr}k>a`7_6_n!C%@F#WvgE?KJlN?ohd;8+O-sMONW`<~Yt9IT0a*5T;;RQRge7N-PT# zi&i0SOBlYvn}}$wA&PXgUkGKJwGQEY*xV8n^2TgXu{4#MKw8^OO}6xWKV_FA(O!s~ zRbht$qxBx0gJYm)S`%1dyz~ZJHetV&&YP#4^^Gq#gZ@#DBa+Oov*&MP=2lljMLC(# z(wJ$YqDGed{*C7Ky0>2(R(Afe&J%U?iiZQcmYM2?5lVxk=c9UI|MJ&ewYp5v`8deZ zhTHy)Va-_uhbI{f!A7Vu(qoIoVm7Y(Y9n){YRF@znsualb0r>5XHBW#u-nNuF{_+$pcxA_Q1>b{hn9Eg zR7C^WSDDJ7QOuJA$|GaH@MCrpa++#96YH{?;JV^*Jp? zGdG2Uq0`@FN1)Ry4Y3=9U&JCWifg#pQR<>9;5J1~lc4iWVuqK!ZXi1oY?`3Q#^1c6 zp!l4ijg_*1MxvM-Ii=WhEzKGWtM$AZhu$%}1s@@SRfyq!9fTYFkN`G;RUZ~=9Gh$v zZ+!R<7}Nwd6(*(c*4nU8yRGM4nhw0q#TBCKb2op?CjB{*T)GR^QH&6NnS4a1z5}!F z(3(>jf0&Y5gxRbSn-iJNF_8r7d8|O>e8|^IoH3o7wn+Tt!Q5e+*mOn>&Qxx(&a74B z1nB$(MM4paB*oR)n3$P?`X&yGDbn$fSdyF>p|q`03^LLp)~|=FK`xg0sUN@Un4u{$ z3DYBxq=!5d0+FmuGhI-)2R8XAm5g|#EOl}dE$8;#C{noi(i@R_oK_4@)}jBo-?bsk zmdVIL&A4nhdUW-P;-dF$CL%_3I9}E!^AVP@;)9xOG2)`4ehwJ1L!ATZY%x+M%)gke zPCd6GrAS@=2G?6#!o;eHV#cTyhOKF`TbEsTa&Kz0V;FP(h4*iF?(Ou!<5-w2oz92e zUJM+WJt@x^#(90cs5V5e6j>ahNiF42ua8*K9F9?&(i~<3wu#S<8nMx}FvnpZXj1%Q zjyv<*ita_?QX*Z_9CHpC5h;Ni5m6Vlr88w;dfv{qqc?r)iVY%7J7vb=%6$E-GTv(o zP4kWo9n|D9Wx_x-q{AB-$cJgYem0TC8yTukA4}(Q<-3{$jB0W>g^Wz(!;MTRiP~1- z{rl6Z@qjYP5%dGF+jHd_s;Ss2)=>oUq@QUxFL#>NR8f(4-F53XDbg1lbIpeN4a+}O z>r5_S)!GY|OJ|8gz3tE#0%nMppl4`k;g~Anq*Kw5MQ83NHM)Jyx*kQNXo*eH=ndsm zIj-+N zFYQ9`#uQ~mv&O275vNnrue7E@BlI+ zQ>hu`GM~$pZ3Z2|Zp{W&c3b2A7}x+GW$H;@b(_*lhy^2-V&npZ10G$WARc7CPup#> zI@ePD)X6KJs+1#37(I>|$ZTpDApfT=2qV)js>`AMi~l*7fB%R7aSXV11&OM!05p?G zP(N4nxbeU7v{(^^U+Qb2%Fr~+Qpf9tB$m=LB&=GdjxFrexH|zQUOalEJMaVI zKQ=CnS=#2(;YQQC87emDX-+j-mE=b~q}?s`+pUQ7-eI2x=|0}#7acllDLwQ$K73Ao zI>c{C+B+P|4bP=o4vTHRNjSe&3y7{Aw=vV+29yYY@Sx|(kc4a4% z{OKCWKJ8vswX+`2-P;Qy>F5t&O8YAL*`8A&bzZ0<8B{x~A&QhTe{&GEsg{*oj@Yz& zooS}GEsG%w+^mXqWzV;NVFyhh+w_AQR&3qSi7@iOEk=^Nsl5EB!vDsIi6!aBA6HpI z2({9)Vjx=EC#}xCB2-%PFfobqZ#U<%)9PQlr1+wpD7#30wi;!aOX(`cji4UOHx#61 z(q$ZL14RC%k*mk(W(M?R!7|rVpeK&`bpCHCC$aiX0hxcyYqaFIK!5_Dfe_M16^9d3 zMi9Zvb3ySE>|sCdU&j|`7Jj~_%l_TnK5Y4CC?B7sGe^MD>F&{9kex+RHYFm@MEKLc z3jVJUe3fwjbH_x9H{_#~;{)?@vQ2y1-Yg53zjOVJBtiP>(7VIu6k9YMf z8U7yrY4N#d=<4|b|FCJ^B>Sx`@Iou#s4FUEGvRv_>~F9i*^pB4+j7>DeVY-ES&B*u z+X!6VK9TKxrcW*nu|R5W$%`koX#8)NAklar(c#Xqtc@-_RE}69^r|t9PolZ83#WUI0YB-(}kZ zYbv?Gn1BM#(Y=1e+hDWRXemQWZCsFBSb~GugYc8WIw0jhuv*<=3JmkM=0+#}YvcBx zZC~_3fF~E*0f0e-3aQ%xIL-}%nJNJssIKcdiWDnu{0|)nLh+ag>fb&P3wpZo2&CaR z9z$upH2sjp?0;;am49yyS?P}Zi)>UFNpG)y+}N(v#}V}A)yKgSOn@WQU=!hzY0{K_6r zb#^JA(RY6A%xbf7wG}*tz?)GFC(S|$_gKQ7xI_DASlIa*P)YG-8L4!Rk9dK6_*-6p zE)`BX|LfXbgDg^Q3M^6~CvHnEqBkT5(r*fxSF#Fxme~EDO`z`4CW8Ve3>FMzeUy1z zdmbxPHx0?v!d6<$mStz1kZ4&j5Bz#Ecls2DR6D<<{4U0j#ry%&eZRPI5SMdgSWj(a z?56g8B;3RU|YNUIMs2W?2xbD$iH}c=5ah1Ecv@<(xVt(R+^D%?c zzy9>!LH;=axq4$62aliX5R4DXyXLe{f|BdzV~c_$s@9ieSail7nvg(m_lrIYxu;FX zi}_yPuW>`32X3LN4zScWNc!5rMSBDo&T+m07zXIi0MFd7Z9bn44`0=PKQ|Yi ze_79$`RqV$PM&{s30;jX@EJAdhCICqqW8q3ysTb-TRH`Ff9dz;0iK^FBmg^75vq7* z;|VmEXWmiIbBv1*f=2oUCpJrx700>ZJXD4BEJY~EwN+R3Li*Op4$8ox_DohVepe^o z{KE(~=p}w0XvA`Z&w;Lw$c|dBmcT19T$Y`Hm(QxPo?wq7txL?>d9y8kG%VNFx}`+7 z5~^E1pHRnN94`6)^!{USaO}l{-5nGC{n01CAVdF6iXx6BilRW<+D~9S86M>M7yhcY zzEz422HGoa6yM_hQydT0Z{(@B{Li@}xi8=BJ2g^8o3uO$lENK4w!QiAr@y@_K=Yej z^owR`4aU#`7IbItG}SU_$FRHCJp! z1-tGtNeckC;*csQpN9+zBM4gT-@Awd9(IPi;_cJc2>oEq! z@Z9+RS}Wv$PHIPK!ITpFxxx&NI<1ZFUaK3eEaVdHIxm^ofhRCdPLL31qJZJa=kZMeR&Xzg6kvf%)@dw^qr;muA?i2Dm;M?|7r z-gYzgPiwD?oRuR=tO#zi7o1ADsm|2zH|VaykgJWL_US)8S&ZV0dngJ`;pFygbDisEP!?;+as+r_26=)NHU3 z+%ujvGyw@L?$aFkgD>W-{+c9yoY<~MEuV)>reA=fyXZ}&*=Gx+(7tyCUI~ob;g|G- zhObI?3^1?Og!c_^_VYYVA?GIEWr!m25gx{dvt!%?QxkKq$l)`UR_Jgv11~A6XjGd{ zrD$KN5$_v0Vc|os?}yaFNb@%s%&VTH>I9Q_ols$-VUvaIFMcy@V#YO(%@TRj$^hJz z4e|#71nL>;wkJ;6B4BXd9~tl{`40SdiMJVi0&-A_C3z4OG5y3;&JIw{TBDxUqd1ov z77w(wgQ1gle$kUOV`mu@FZkO{cH*>XUoM#)2A`}IwMN@lH(9OHTeebm@%28=X#!lf z_7=9bzP>r;d|f|BzY7i;?YscCq{>zRkd2M$q9R@<12rh=NHM=npzApql3)BQ7taol z;l)c3+4F0rot_(#WO3#mPnddbl1*OH=|d(4R?XAO3igu5tt?{*DTeaPza-Os_LEZv z@_hET0a)3d1-F!_Mw=Q&QwA=>-q^VfKXoI69zgi~A$`Ll(~l9g{Reqjnju?oeLURz zs{Q9;=#T%a6I}-e@9-gPHyG`6_anW zIJL&)1CAEVOs|LYQE&S0<7=s(4GT;l{zip<4SV_ zK#LKPUX)~8%m1989?)^8G!7D{LSbrvif`Y2?~6%J@z+r*x^%6jVT&|N)amV&BHLYC zj*GXQ7KBa9H&1qHiC88>T{MY^FkjDl7IJV=0vZ8*(Ho#A-nFx1SSLoI*LD_H#` zqd|ai57O@=E;p*qMKtbI#%^(hdzUK1+PEJq*=8%^g6(FoWtWg;V1Wcx|A+n*U1v;L zZnklaRfT1k#$>$5xr_OylE6J>6^+H`P;$FA-G02 z=BV{!a_GFbQ|Z$J@6Udm2ErYKiHh2e@;MjrS0QyfP7w@OYT^orZd7||mb4Gyd9r7sF9P10D(O{qWj_-@O=Lb!NJkQ` zjd5GnV5)`#-iP0NpAZqJUvqhZ-@4b6h~$YGCE{&b1i?wPuWc+3hfO-*5Jg}7qZq5g zh^9fG!7VE=T5k8`#!%HVrOs-YfY?&!o!l|w8qV7aJ%ZHY{Sc$gvpvwG-({T-6nQPy zjJAWBHl8L3LcDTd*>qmXJ7Z;G8hs`mYiqn)o84?F)qz%@uF82@be5O8 z1SzcC^a&KBt8*BIGHB*;M&kQj5K~YQ-3GiLV=l`=P;>Acfol2SmheDrF*^(eONo|s zT@jYD&yK2GgftlhdCq(mR+R)LPnf(_DT4x-sxnxrf;XFt4yoQYkFVq`mg={}IEiMn z#{e(;+u>`*E=Hw0%v#lWxxuN}OGTrn6B zRybc40eFKx{%qxiv9ov8I{zwN^6|YzEOQT+yZ?}R@|bXl_vXUn8HC+N{pXjU1oV%#!ri%dm& zD6pkQ3ZEG@_vj1??|r0n!sRv*a38UJFULl{rSKb@#XoTb@&ei)x&Oy7d?kSxC9hzCo1j^{|$50nO z!qA)#$Fs|gsR;4Gd6P{X56c!7H#qQeYSD)b7vKx?9n3##71% zZrdqd`^WOG=eL2e<^K^LCO2T*rColQJejPEVU zq5E9>F4gG29>Dz~V|~>u>Lhz4pDOsu^A7&rzVWN~uqT#Zc7%b1l^Hs?aey<+`%81> z8$4WYQCZmzm|oQH>o71dKbYeqJB!{ymod9AzhBWk7p4MZMI0wyu(N7}EFo*btV{3P zl8vXc*uNzLUQN}hHn~kT%$D7l(#+XG0w2E~wsmg)=DFN#p46mRHEM^MDmen{irHG7v12B3w+;Y1X^I$!_8`cU!fS1NO*9wC8d}_1&sOjs` zkYI1WBR&BsGb-ONNDFS1$~zdkOb7uiz7kR9a5iMlg6G18kD7a z^~Vc%d3k#G06=$vr9P%%ZoJe0*Y=MaG~3Y%hQpXc>}LSzwu7KYE1>pZ=T#%?zP9(h zu{IwqAZPDPBX_FDILp5o{K@TURKs5Y4)A_Zc@p2Vuzo?ClM~P_gz9keDlb0-B9taz ze)|$VprlV=-wa*?O(_qxvxpNix=N>ej31XIIIV$agb3Vj* zC%RY?E3*dRcb;uH117k{y#qN@HOx=s_SE|&JpaXsll~64qM*A3)C)BItjK80MR{x2c;g zI!>m=oy540@U<8c_>er+m11a50g$EMc@58x0ef{^0Ho(|bHKIyz`}jNF+a_DSG-su z*Zm1UfaSXKc&FHQz&+!H8WnN*k1Nt${rC|Eq<27@TMvJEhAYO$%s#L=nodc>ZFy5e z&m=S*yc?1*r*>^GzTYBnFtI$4hHR>9;C znU2de>PzE+ja9xzn_uvh`qM;o#R@qJ3BYX1cMhw?i*RF~W0d7o#+lg3RR~iD^Qhkv z6;LaRXPoO(Z~OWp2d{!^^EN#IuK-s^MLq!Y)+Xn5@3s_GID0564nmzm(iyi$j>}u( zkxgV+16SF+u7PRNGOz^;W+DfiAUJaO>xum!t14;um?M8<5*a0E19UT)-3xY6T-K09 zMgx4@=41~O3583PF7-I4%w4IVjF%-O`WnzW-f>mrMIT%QiX`-eJ}>L!+AMvtGc)0= zt%=*(RADBQW^Xx+IJ3cH8Zf2~GafD97%EV1uGpqTN)3hPg6;!fHBvmz+BmV&R}D9Q z$vv^EG~8raCXP{wA>F3?>Tk+T_KQb+F4=%&E!bZ!2rkvcPXb2nm8Rj9>8XC>en!3f z`}$q=G1#aoJ%g55`*LakXyZqDn7MX;R+fHbhK7N2?p+=7q4*pFSTE%KN%h>&nv~?rNebzv$+r&8n6JLlc-faPu8S+t9qpzmN?mq6>p4TOC=&WQ^%yX2{bu}y zJqzg5rbU714d`&U_7Rq?vrJBZoD7XfJ`bz*5285G2E?}m+j(OnNz*DkW@hB2c9P9e zW`enEtCaW2l9*r~oUVQ6WNr#&S5fSq_DfBC8f?^W_o%eU5AC;m+m)`&SJeI85^O4C z66kbN?cY-^)jtMe1DB+p@tg%X7%2M9C_ny0dLXcs&fy?^*h<*Zed&9*wEn0;5j!gueNtEo zvmb1*P&V!8nSn^liO}aIpr{q>vSEK8icHOMW|$L8*#nc?s8$$EF5X|6fnPV3NGi-B zZz*d7`6a9+{$p;?CO)dcl1gc9tmMjm@1mlg%k@{ zkQaq%^E6uE&bxeRK^>eP`XjiVhRRS`pZ@`-0o(fGnZ8j*J^#KbEu#3-#FG; z6*w0KD8}9Qmjm$T2Npiro4*03WrqJ-wmB1D~S85c##8}%{8E3pzBY8^(=pdaE z#pc}9xzabfisf>FYqQeWgiYgZ^(tjGIqgew3r~9a$;^cs2j!XDWwtA4bLpcqZJJt*3JQG`il*RtwiocjDYz`_{QU==*3`md@r{%U3;Gp$s62x08({F`ekT=aU` zBbKU_gFmG4Y2(4*)KueL=&cY~QS0nfI~(wy24H0T%Pk-Y0WDb{s^>UB6@r!~m!~$e zueKEE`?Og%Xh%q;#A`oP#%lP730Im)XV=(QIG8dpsBdM89&6T{b=oS*J_s z!h`OXb1{i~ntFN!7h>V!1P}G;e{30=^gwK^-$H0FLq#)GAkVbIh~q(r2Ul5Fm$A*n zsNo6UT+kVV9!AadxglUvHW71}}VqK(5P#MHnbe?y%?5sa3|8R>B_fv04Od z8R&CUXxvmAUORw-{u^bb3QR-pW^b~NN>X5*tbw$++>&JPQSSvWFI4xw!&%Mf5D`XP zBwM8e{ar5F_qVFpdhR2r#oSGwee&W6$s*RxjD42XwYec9Z~bg^328@MwoBur(0E*n zxw}HPo{=rYPTe8}GOoi2;SRnVR%_-rxlXCrYSUh#h|YwY**yJ;Z|cme67Li-5+>MR zE5zuxC^9X&VPu;s(J7QBS7@u8cS$l0G4eNU(FTrcglRQEYNg1y13SVhkE z(zSlyMms!R+iu`D;D^DZk2~729_i{cMro$XJ^;EVJ3#nn5#Km`|x;PsCO zzb-l&N=q-0c9PQNzrwS~CTqfi$|71%KN^ljMeAve+MUk?jF6l_`3*D%9u8QFryBLG zorvW!;dmu)Sqcwr1{>6w4NB@FWT_i}B{@!Q!IuGmFC10u1Dn?eFA=|j8pIc@wus*D z4NkruhG2EUPy-r&!~guf(9b0@rp885@prpcbinAk;^--yA*sqFb7G zId<2EilIQ7(icRX(s&Az*wkB;UuMy0?l-LCOntz zIChSE?iF00_~UA|W=GhT(!?obUa`=w@%7p=n&_j|N|Jj?4|AE$->EumQ%*`#hOg;M zG5nw^(O#R@0yBY%n-{R@BDSVVk52DcbaDY*vqqVRyH`dY(5t(*@RDja$*S9Kxi|Hi zixK7F9WJdh}D~9DZ^5Cnj|&wb!rko>QB}j5)WnVPH4EOa8S9%ll06lU7Vx z(@rK1hYU93A~FF5A+l?Y{4Y5vUt|&D2BUtqs#F@}t|9&19N00j-CPLkU|xeBn#7Xg z>Pd>$u1pn%0?9JmyV)X3wNIvP8%$9G#y%G0%RblCMVu~htS06RGY6F^@#Y7xF{etp z9JbPy`e%!1ES53)ZucTD{--6Z8pd7~VnV;EqmAn6i_|L#2VVf^Gw(d4P)tYJGWjk$ zFf;-24SM()@C{Gt3f1ZIxCELw=4&o`D_d)29{`@^Cx^8TxX%A>Sx z3LnZ?9{Wqw`r!)LIuz+Hpm`5&f}i`2bO;4B`O%jG2uf6=|GY44A84FirnDoz0M+yV zgtFpy^qmp@!e(9s4&%KO(fzjrX*h>};qg!MD((36bgJ*w5wrw&E@7~NK^a59txgAO zawetWnag87*_KrqZZcX+%Nw4DL?ef z0dS*$X8i#8bDCEMcn^p6D+FAV@F9QZDkKQdXDhUPg@>+|U${x)?sNEV-+lg$Iz*>P zRm-J#!WIBpZ^V>6S`DaWLYlRMS!7V|L=!B|d!H_fU*QNB$`>_g`GWh8XXN6GAtCQ5!qv}~hGEwf?2 zr9NAU*}k3WZabXwH3I2ce&HKN!s-WRT!E-|02JZ(`?_7BDW!8?SO^5_6z*sxng#!T z5NF3A-PTU@>kh(TOhVC?+VN#Lukitebci%3sNCap$zTMb8ME$zsc&u~=I}v*CM@8! zegO7+DscEHFU=5cvfYR3L$%);yYDqye?sgb-WrOo!5PTf&(+&ga`1$rhv9klCgfA4 z)Ya@YJ(5`62`EO1`1McvCrupddV5=YA^VamLl^Xp$G^LN32#-#Cr4e^N57zBj>%kO z*)rrhvx0DIb!V;{j|G%9&T$y`#1OZ~+{WjdiZah)a!2a5bWv}XwEnWa;Hjl!(*sBi zDp}0o$9T7=r6*%5KTgzne{$jwTwJ5C1iTjp`!#-bMt_HO`)_E*4@#CBcljsZU<2ET|t zCb1tV!v}l1d*h)P!xup>yb^Y)_kW(4)+TvGDkIKQs=jrtyPS=MNR4{r$9_TlK^Dp= zr06^?{B}}7?T!pQ8xLJ-Hk1}!{$62&`7cAcq#|{=VtwiU|86KZRTlq8Pfql3zFH%q zeXfmGU)R|C#@IM`Ofu!T3s{s|W0B!? zck>8LIkF&cC!-Ra?fvbw!dItadDvF5k%rq$c(38)&h1<}zay@@_+qwH7xZ{zZZ}=l zF4sdO>nR>RjG6!?@z+?ADp&oib4sFRV3Td8TU_9i=YISOll58aU$? ztC`6GaP(e8oUTuLAsJLDgw;W-$<(oOGQry?vS+=hRN*h$!SmH77PZ>fuIX5cAwWF`8y9A?FH{vuMc^`F(!==6BAQUf)W%{z?dVkdttZ< zvzKTqzlb%sAO2Rp?N|zZqi!+H?*JI1I@zg}o{{RryJ={mc3%(Ij#cT7doM)LkO6h| zcv$+#153>zoXOy{ z9yK(G8^JRpJX>Jdd1Xl-X zq~gS?iSY%?KXmq)c1hNBS_#r=uc@rtK8JGO53VERq)e@CIrprv(!H)%PP(^Z>drqX zD3PVNXqk#Zomf5&(bZs3cguUATxnv%nDwUafWd7)NONO0Gk6MtNsJkjQ+c(8`(ed; z>7r_9ro)4vh35!*n2J_UvAvSm{E6kw-p*=j1d?j{Zai`U&c>0dB4i^F124SP?|s6 zHwz%eWPGE^V4q6T9V|<^p!AN4AU$|xN$1b{Q)S3u3z<~lAICsI*oZV zV}2@+N>+oLz$B$#QxQ~SWJ{{0TNd>25hi`Ak3%#;B{65wMyBC#4Olbtb-1x_6s)Y{0$)-bclCO1HS}gCC0jbV&Vtj(H%$HdyGImI+*T0q zQkNT^cW{(g`7PVT?a~%RvY9y0Ajg;H=bIT0g$rz%?s^)6*LQUrOy!!(Kv7ZT`vGc# z)yBf>`rx7boJ|W8?{gi^82auq;G^~ZGJx<@B)_M~ZWXaO%4sm|yV@g*=QbVrl=8qo zi*ZUZM1GM;Dym!3LoSCocRqc&mVA@8Jl5dQqL$o`+PoutER4mh$H8rFz+wg?+B8Pe z7v68u-Q6>^dZv?X3%uT9Z2e=U1207#ZlK>)g8`TH;&)+%*}61(4%5iX5Q_c^IzybR z(jm$6ML--M(n$ zGPkoADFmFHpWdo#{M(695*3pm-~wBka~_%jm>_=~3a$zu1b&!Cw!t%{&rTUK{ZP+S z3;I-tq%y`+j-vzNuKN|@M-&(l>hD($3;$-b0y)JuTdhy|gJS`HTZkOjP?CV0!gwkn zi5wj=xEyVN(h@GXM-fy1ConN=NgX3`z)3JU3|?H@YYqI!NK`5uV+>H}8=&XLf;F$T z9$T0hK;H;3}E3R%OdKnBG+6jkP#r<=b#2Rab9`U;mrIOvkJ`S7{KTEQWJ2 zb|_Ut-=|og#IN$-d`xLZM14yipku@PM=VV-a(b+uO6eEt=OkL+)o}b4eG(Og446Sc z#iv@mD3iL)#i+hbu^PQ~f^EWTOS^9csW6EJI$4rNqL!Gl-sW6uDXRrusixmgwxdU> zLN32r&X*I5N~J1zVoXSIO>r{?VFVJ%8-g!oIm<%%Q67o?Af!O5q?-&u)u{?-_1>AC za8Rj_%Szub$;{S+N2X1nMqn4ArP4~m@33PP7qs!lr~@E#R-^v`xX%apr%nTWoSCsK z-vdw;B0m|A+vbiH*828y7$~9$N78LNJ73*f(*>4sHJ);lH?ocf{5i%@feNe{IN`WX})>{)gq4PZDMHi=>& z-aD6lD1`5{{#G&}g@>%=5nk(~WNwzmTEB}myqB59ju)yFU=ONs4r39BEE$d+NjPdV zdHK_qK>;Y{|?155JkSAWt2n|6rxaQOlQt;5rYEJl;Djy>(9_=_*37|5W7JJ`H-+P&f)1~L2) zM?TE>K|bIPQ+G79oH%&9N&SoVQ7`NizhDG{MYLOm>ZeS~Nj)a2sf=EJR)L%MDlq47qh2tI6_+MjQ;LBV>tq$}VK)NhlR z_+lI^%L!0EFW;H?+(S1q7tE=Rhe|1o@tOJ$V7_fZ^@ZWJA2*Q}{S3u^WiV)pd%6yGIdLX3xvecQ{J>jCuIYmc%bzWL#qal zQjN;w+|T8Sr{Bz#Cy#-_p|+g!d(l^?nB>JUPPf_S!Et zCZg^MDFyG_U?lLUSb%gHV#y_sxSnk~y}dTMpH1JW;icjMW3F7hr!bfsCWsJ#^MwfZ z1v^VH3mP!p{BE9~>n?Uh_(5**K0@*Q2*Ri%ExOf0d^HRw0Q!4E@uyuzn#srQn_G-M z)9+usg zC^u7u#Ur0@Nq(s@bByX%jVeWfklLZ;lle-c3n>{8;^yA)Xqh!k;QU{7y<>D_T^p?( z+qP}n?%3(rHo9Zm?AW%Qj+2g^if!Afljj}d`_8X3cI{ugM%7xQs&=h4?`z(ZsE+m) zCirO}uBjXh5pEqSbY2FEIlbuSsL+RH&{`P5VkIS+VY!S0X^2gT1!3SBo@x3AaT$-6 z_Uq9NoFv1>5dOYpaQxzTPsN%yDW%T1Ep1)N-_*Nsl0$m%TqN;- zwpyQlaIecl&Mbx^54yeAaVe2~qXaC2?cS`!Qq*#^k%5p$uW6rgNnkZUr0^s~}b zQm;THm`UxZswlj#=5IqY_eZNf(CCRP49ByXH22ppqukvGN8${+(_{nvRb6JNA4Q6b z>%$W52S+(2vA8-C8Vi~!g`mwSC|($qv0Wjv+L*j{j3j*EsD{d_!G|QqgOSi6Kycy4 za1m@w7M6nj+=*r-SUVy*=9I^B2-fY&F6FJCMBdf=-434B7Dqs0;Zw*azV;mm4x zK_-LUm)0~!Oz8ADk5y(4xAk+-nY;Q8|cFdYB`BTi7o5h)I z8dSQR`|8IPTL+0>M_4P-( z$7Vx3B*w4>fgEL&5^)uZ9RPXH*SBk0ECVMOYQ~#{$EylARHjhWoHn<0vmvE&A zyz6sRp~1Hlq-uad>fV`s!MnJcB;MM=z-%vq)H8v+P+<5@g17> zy!bb~k=>j`h@ox>@3Q^klZiL`Cx6@5ONFxjT4NLa8A8DEt`ZD>^Wro z1Cj{ysYXy?@Q{#=w?I1hx4j)qs0bde-X0ku{5u5v^#-cvvEq$#dSg=q>0>8UL@a2C z-hDb~N7^84pA8boZ)u6{0adnhYEr6%!35v%bd2{qKyB<*0wIPnZe<8!9O00Wz6b;@Q%{DDfUHtvh4rE9Yre)o4e6+Imnxuyrs9XFg46;#NP2(t_Rj3UO(yH=hSKQD7=>Y<6_cd z=q#Xo;&lC_LBNGSnF!Z~+H+XyF8~y24xh9pm~B`#PvUcqY9nCcSlqL20MsS-2f zRZ^j~zlrmwAf;IwfB}yS@0L96XE`Fztphjr)oDMGTL3sQ7w>u05jm@Km^6cUdVc)g zcPvVzHms0}8Jdm$xQmo}1_R#h=%KtrhgpxQ%=B6|sur|z6ys9Zs@Ef|ehq57y`Yxx z@J!T&Ff?z7A|iSwdI~&jjxjI?HjCLF_^71#w)V9?#2)@xcGOOP{1e}I?PR@l?m6pV zcsU3&%jldkEspC3X|zvn&jhimwta`~NKO4Qa8)5#k!{|*ZnBg>$$mp@TxPqQJqGS{ zUCG<3-C90Ys`vn`Qq=CS!YT!3); zkFSt<37gB?e(Idh*nW}<+y6L5HoMBT`Jg>}Qrn&pssaF41t0F9`s6=>WXC|_Y(1+_ zfApD;x88J9dk`ydqj#vjDZvLkGth|8B87lKcF>`}%qHLwE^9EeQX*S05j-|>nj$nF z&x;5$$!*)e`)nS8J7&%OC|YA!B~P4kKz~1%BthVy7eY^hy>euA#@nhcKsIca!y^iPBi+2+^$sqnwA%9FObjg)W)Pq~?x zB`FRI13weLlu*fqIPV*P+mjE#ylnk!;hBHH?3|PPZG6w8S>Pb=(p%5cb6drCmE#44 zV&o)DJkC$;ZPGuv0H?$Tv4y_(Pla`7AW_Pqll$gx80?}}Or%jw80zjYdz6ZV*Lf&O z|5vdvp|EYemK2MXGa2vQ+XfE>-_NzTP9j$l_YFPT(T0I67Wg$=)$xh91ioOxsv|L% zm&qHdgaYZ;YruAm?Sak;q5o_7)g-)&{-5(pQT()9G)#@EvGx$E5%-hEX?X|fKqa`n z?uFI+p&Bs-X6>5uv7>~Y6ETY|k*-7fAAxpa-hCGZ&i<@j{Hesu%hB?|!cNz@r^P+Y zxqmI^U8+>Nhdk4x*M9f5sN13h&is&W*7|PHQnJTmzkpH zIf;n%=)Sa^a5~&k84=!NbI)G3`^QdCwW+?IKYa4{{kiWZV}OqlfIqI)gMPnOuCWIE z>LMRS>$gFr-lgP}pkJ?i6C5E*T%amVy}8|KE^}vfvqY+PxX5@9pd?kxO99(ad1yKf z#ZS(u8grm`g5nH4RZG$bt1BVg5p6xc*)ZG6-x9}?<7)g7hI%XO%@&cY2p@A>bge_Z z7^!48l83BH5mMR>ZTWY9&PQJOR9|XV3_J~m5fk7ddyaSF0asb#9^kg6>o zCjFiJ!})Un4^ilGma;y6jVQa!POU@BX>dZO8?vsh-@ zRjuSkHRuUZ0?8Tv-~viB-Y%20+v8!+`e-kaImXAEej*$)_8hVpTUiZz)<1do2zxru ztQ>oih#{SmZ{qnzdmLzZ`WT;w`zk!moNH+I0W3Q%;{!fRZuEw^)jMx!!^z_-Pz%YmN5VW?AGJzgf6kF-Q?ag{=#RUH~1?B2OjY1 zm%;?WJmAM6?o{TNc@oB9QT`q}Q*_P`>%G&}y&=-6y`Wk73jZ79z#_Qy(ISnZ!Vanw zV>QM3z3NPI$1M&4>g1C#qh^|Tmh`ne9GZ4{w_WApXoaHF@S{6le_@4WlH zT}bXri_(X~(8KkxL>=3JZ{mw1D`0jIVUAy->jue07qU(*iYCM=nbsi zUcTMV@)+OiwVZ00hAJ{0S;pgMit4_*i8bpZa@n1g(lj^gpv&iZ(g-qkXWH^vTAO!1 z@x~61(~mnLY-aASa2bFIJZ<_Z+M|n7k97SHszZEnXGK(+sv*HS{N#qMl77Pmey^Z` zJ{4O$c(^S_d?!F>MoyFQ%0#2y+LG2%t=*|kHrw|6 zM3H(b1vn3{T%e}AZi=%E6zpt;$dBUFp(eBr@)457>lT{h+q(_(p0MJQ_+@0Nebm>))vMi1UG4$h5eUR@M~r)%=B>ud7G@fCsq2}E*k%XXjo!Eh*@k}1 zTbZf53f~@*(i0H3h1aHcA zjXO>AWzKwEvy{dNZf4fg!~vDX61+}9ASbSK7bc%nfmdSa&Jo-iA4qoaw#khG@X=TN z*FuM}yCY0Z$i~5OXC&?E& z6F21utX|u#5LBO4Sil_@Pf|vt+zSW(7*7d#N}R&B zep}>zLJJ?&ub%NQuX^|Q#&`EcJ*Lh|xrg`u2)<`%1Ox9skJ_$G*S4xT9j^1UL{1-!2&$+y@K0h7!`1wIldg2kyU?A5|+RYu$gVzh`SP<8kNT-3ZvP|=JM#{fAK+G$;Z(_E|CGWYwlv6iW zl+fy3@!x+iGF_-A=demXR13pUs&hk??E35U!aPPtE-@c1GrI&sHTyLb`>C)LLvztV zzDYVLBZ>5;bLjV`TYc^+Km7MJixX^Wo!)fA-ce+{*|%YrH_5pKqLuQ4$LNpzi(WtH zdAH9!iF54mLc?sUPm@$X=Qa1{Al9}Z6&4#2Tbv1NoKRCQqroICRyZkh;I9~jIYf0f zRSYXe^4sYzYuqwl=_=f&MFhgIhJe(9K^DzgvlEsWFv5M1K^Dadr{^b`FwJ|N zp&o}I!oFCzF+r4{vonq{4yCrEroY9nGRTi8se%Uvn+%?9NoOs3WL0v)Ir9uvOJbxA zi&AyQB{}b6CFoki>X~{{WVvdkC%tNJRbgh#bM`;K7?+M6erpYPE-h(!iJj*UdHrU2 zQmE#>e~`=gndgfk75&(ewW-qJM`CHLNws826vIo?k&7Sei++J9=%P%uYMU)QTj+ab z;jZ3HJPsNH82f!`#4K!H@1EpaM{vQiRLv^pw(3Y9AAV8g{tGCzr>lJD5 z8A%uu`geQg?n$`m$8OE`t(rX$Irn*{>DfQ;{Jod^VX+A~Oki7Ni2uoN_prSZa@pR^ zteMDlhdeln?V8hk5$pgT=%&8%H#Eb{qa)W)zxO>uRm}gchILQ)brI6A@K892OZ-x~ z;~dP2e4JU-xkChW5XXn06$?C!55tMA`BolPb*Bh)cJ zA~tqbSoDp0(m(iEY&>QT1H0wF^~EW|`gqKWRG64kkl4xkQqh4e6gnH?ypg~9_sj|# z#7sl~!Np>(k3^}3Vg{Eqkg=#n(Pu)_E4t&+D{{wRV&l={1z_@xo5tr!?hB)31)1XV z8>JBe(|>*KUwm9Jcz^AGd`OJ=+wE0E0co=KHzOe;-EV$Dj1ox%qzrisHC3-HZjJH8 zHv$UDg`Rl#itLt>8;D6zW0{cG@yDi*C}XpAMJy7;-)O3O>eWyji3x($4}R&wpc_!kglo%dZ&Hm|gLq(;=wZb9|QPE_6uq_0< z#_>_o!Qqx3OCO7B{38qhd9q%G)oQnR7UG}a9pU*vy$`ciQ`eiv=ij`(QwBtqu(8lA zLg?*9!0*CPFl6%|ED*ws_Cm&-Lc`Otw=U)#ll=JJkOY&V@A8`=lo3D*aB>%6ER|?m zZfl3Dov=aQ|G81@WSbIr9~Y1#O`lW%`FgxsU?vvw2a_&6b`K!;r5M>6d{5u+6BhP+ z@v;ASyI)t5wm+oiMcbsg63x`-?|b-=bV5U)s?OAApUnKVTA>cly3Ml9iuS{l^PVHa zlg(^AVEy3}}7?q1_W}<+&pLFgXr%_VP=d0^WWcbUXna_A2gx z7DpcLAKiSQqXuujk_}lA$2L?Tk#HBS;yg{M&EOt+gnb!fOs+4Q=2YsXKhzTg6%$ZD zLN}d%jrSca7C^@<$5~l8fBp!&F6xMzyc}2wnQwlkTm|JW&cdpDhoUb8K-37L8`*2> zQf*tLR-sRcc+1lqp-bPl9$mKF{$>o*r3^=Iu(b}X$?(dN*Z*f4ee28)K%ScP9+%-%HsL9cN}dhh!`lcw*eUv2!3#t-cSeo-YVQ zy?R@6B2Ue~e|wXxh`R878$K;}UyzY_&-fzL?55kl%%wg?#&-7hDDZ*KZ+hR~--C)6 zpv?5`m4yFBgu`d4SINH66UkK(LnArs3<(%n&D(e>mQYrT`1bM zwq5qEIV}Bg$)+uECQIAz?eEOyE`XgGJrjV*kCyjmE(~fl#uAIKDm<>k=~0;}Dtp}zV6n2z6x%h>l0eVY~+%iJQoL#Vou%0c* zKXa+1gJ1iDU#B3(Ym&BET#G@n@aLL1t$uD(ReKMafT&XJsBUSKzAgbI@c=!Kx25yVbTeCFgT`yqmM&dl8WvAi&HxZng0xq*T)-{UU zf6DZ=2W!g+HqIMOwf};qr4i%c^$Nr5UQK9KNuIdl zo-C$q#7PPr=8UmN{|DxQCUR8%N&`)Oo5aRbB3i0!MSe4wjZwGQmy7YE1Z!^$=(;l) zSEPiEfHooQdydHi3Pk3)G4{Q5Uv$D_n}%My6suP3R<&&BGU1o5Zv$hx9M3>3Qg?$FmIppaT0?lxf&a(z?Q{}<|4Nc-&pq0Hr+IB(Jn?A?ym%KsL zH7CvMw!@h?RjgUPybT`E)QW=$ANZ&=>z??p8n#(kXMW7=`+y294|yq&W_`#;Xz^E5 zt6B91dp0Intt#hA_})X_33O9odH8%h@9+2a`auGftMTh(NP2&QhaRh!7*chLK&We7gUkt_Qcs z#g3Fb{m%a!W~h}IVdh(LwGbbzswI7P0PxXfcZ#K1suIgbpyWQt%TO#hD} zXmum7#^cZ%unux(3t;U^Mk*+;j_zPlJ>s`@LX{S0c3uA^f`RIl@^kha_NyAe0fL$e z1#_lABHpB=wj?@&l<-IYbF)*5ZEQVl=Bh^SYLnTKv^5bRE8Ft;G9?F1x+_TiEEb4@ zrH}8lx9Q->1eJq$D<<#K$c#!qnn%2!*3)hC$&LFDAJU~KcZmJDQ9y2O3WI-Jpm z_-Bv2_Actbn^~#G8U49^Hbyxy`Hk<`JQDG5T%$3jJrPh&@&oklQp1{CZ02%pZD!Zx zkBk+a@^t^-FJ;u{O~yE^$i)U1cracC2q;fX|SvGrkqK{62LDm*O^LN3R?|CogVtEnbK3{|V-aYkB0o25)yqd2`Xnm zjG$CrtNqe06-r`0b_gdB|@Xi;hFL_5lvc(J14VWKXpF+T55j8=M@>iKh4 z&^0fW= zTs9Nt>IK!M#d`t~d&H`Sn&Xi6N#(onCiA5pecQS35aloog_?uP8XvEul@Qvy=LF5O zS)|l!ksu?Y1MTt3scv>Bd_ghbLW-m59PB?`PJz@6i!D{6b?HD&)eYB12)JI0XwV^4 z1#kTKRm`|x8QE(Qv%<$+!QK;$rx*iRy9nK1fa2d*_aUQ+dgW8t^qsoF39gb2<+H+y~Jt8aoBaTRPt8N|@WKT*+@_sJ23>)bPH zox43{o?jN}m_^qy6nnB<}t*e24PWrmgX-{KF2Z3IGVNM~HK5&){_7h^?HnZcKs3U3y2xjX{!#kk5PbFKY8(ch ztqJqz#($i$v ziCxx36UqwCE00?VV(x5MV{D0zoOXB0O|B`xF7`tC&tpg`sY_1jEJFn)cIluYVPmb12`;+i>gtu3jgSgcAJE=9>1l zar%Ym8ZKcJGD}Uf_$8Wn=@Lmzg;FF_cQvmR5gQ_O7c>{2@H9K$jj>u$a+JvCG|8t> zf|i%Q#(_FLNLy*8`*5+9pLHGbVWV1;da#RkV7U6}Y7*kGNmewIbzgnpD z0OV`>pT~N(H;Z#CZw)PDUGxo25bgE|d;}v{G!CarXs3$!^&Zt;739!bUeRWJLv6qB zl=LYytqtM{+mf!}tsMqZzb*{cJvDSz&vQR*{QnY4oqtmSubW=a}&nt(wtf9}{tVihnC+rdmdoBkqo}0dVaPE7NhVwxbHyQ$b5pIZ<&%}T_+WPbbHg_R?k zSrdX?{1(K|Jkh@|`LT7=PJ{!Opz zh|^AMrFCt!Zb8&CQ@7B?&g{g|rj!7GwM)Vja*FQbaD6m-9qN*#ioDeFBr`pHfjr2We># zRj?#iAykJbAktk1K&e~Rp#tO>DB4DRJLg}R7w&(gE~9aI^Wbp)sOijlr%7@QJAF-q z9AO(gi>dGYE{-)Jp444(4y8-(n&~@H@G*DMQ;|kpHQJkq88Dwq$gFBiJbWy5@?Z3Y zA3bn;8gMnu6^{6d@8=4aWlQ9j<{pouHLTZ`U_fzg^{UHEVYy5Di6a#(cw#o#(bhzB z%3!-5986Wj4Wqs|0|sfg>s3oTd(sUwrG6^_Ms=u~=;cOw=wlSe$!{vkpIpZ?o#Uo?KQ*!@p zBpi6r-EKW6buLZ#w}WVU^WLHAZwloTw&^@9HON^=<-$uq%^nsHa>TGL1`#rsg*K`g zx!Nd3^J>F0Sm>6RkFj5->D=@&cKZ|$= zS^>s7%7FMm_pAul{ML=VBh5ksUOS0re>N5)fD79^a1~)y&PP*@CeQLYd1iqsn>Hs> z5@_=%LOE-3)>P;t2FXLY4Yl_uC|zDvg}Z~nI3Z=KmIQ~W5RRZAESwIb?MkZlQvy60 zsuTc@F6*iV$)rF6zPrvwnjA~0x4XB29OA_bo<9IswfgS%R`BT)9;sYcC+wf8JkSxl zdCTmfNB(Y58zEJWNUnZG7HLzxJ7)*G;_3V8`9%*V9kSF(&hiW1*oofwRU?$X%VFT* zBYpJ8WYU-|C@m|hsZ2mdIE_?oBuL{NA;O^59v)rWb-k@B1TJ5cc)Y?4_arKN5MF|L z1Xjc_f+9~daS?ZK_xC3*vMJH^kFa`CA^7g7-%56(?S$?`R;`JC(;qgjZ;Y1*D%h|0 zWqK>S*oI688?_*ItUKt;f}%*92?)O*GQe6n`Nc*LoH#ulyHA$?O1r=JGd%WR9lGaX z2gHYrFmp%2_Vc;+jz%?k-Z0h4I{mxfyWx~%T-p3!?Z;bMHGLBHmS+Jd z^%ZuQ)Zknb|P5kn1#$x>o;JQs<~STcK46 zb!n3+mumHuH)LGYg&s|)c&DRJ-Tb8kxkLGzc{(He4&hMi&$=)H3ZmB`^8QF70Wsdf z?tG7HB{m=8u@$c|Lw*twnZaNm7XTm9m%R5~6~B!ba%Q?x+tv)RXz#LcY|rRcUg~-K zOhQYJVp*KTdu?p`&O_1Ap=&67OU_m~t*1w_%KC!TmcZ*Y{JuI`Bql|QjXC9X1!QH?2$u1gYLnRm_Z&@wtAZ7E5=H67(9%f1L!d-ymd1_$xBM`SL#K;BT4Rdo+fx-5`a@ zTAO>Qjoy&kKsRM{o6yK@#N73-AbY34@4+uD1!9O6v1*g%i*Jzy*rFS ze24vuPDf=iiOgZYqv8^dOXYCIV?&A;YOO)(8-KT5)vNp8w)^oAuOl^~CMQE%qpJ3m z#=rwn!Cgl-=J(+RCKcztnK^qd<&3`~uCyYyS30RzxYe!b?Ad)DHTLoG>V4jf5;N4> z$1znd-mx&dYc+DvS#Z9t01RD+b6UKPMQXH=t0#yh6T9md1cQ%rZ*1a(A^G?%Ojbfs z9wl5kQv~Rfe3trv>EHZa*3#vq=Wx}qME8}{F8!gOom7Ot4iY|e9E7KhQBt!W!64pZ zMgfQKtP3Sh<;-Z2KIGovQZVA8zSe+NsCg~?m5i`%^uHn);XX5QC&Btj`Sv+7U(f^R zfn*Xcvi%~IwAd|Qrh80Sk7x8aK3-6rrd2ePR;j^r?N!2S- z00Hg#-z&Ajim(M)9zovo$NrEPb?{24MH`XN-97nzYtY=QVmW>AGaV34y@*k-U!M-S zgj+#Dcp}_T`M~79AuF=Js&n^Y(Wp6tGv^$ff3Ri&WVoAVpe^ES;MeURL|#Ne7}|aA zMfW9`iJ_4X|5i|Qy2UZ#5-A-|Qsy#=%m@(Prs?MRh|o@U8yUCmNEr%+Lzm72`B~8` z0?KYmCwNXBS5(AehhFeOJB`sH$NXK>xlbm{`k4Dy^%7L-M_W!y68GLV@m&lTcE&Lm zXc~lRSS3bt{y)zQd5mR8j$>H%?K@dQnoVs&}250R}4-*_g=8N?F7;^;#Y&MWcTj^O=z1r z#Y=tUvsrOhU5=tRMK*R5UG287*kXSz$YuNqGcl{dGPs7i`ovM}*pU*t`d@DH5YBiD zi#gXaD(8BBD30_MOu2`4e~u~MAFn-_VC2|pzn959=3sQOTzLk!!Q=W-%d4vzU!LjD zq;w5_`>S>?tF|X)*#xD`7EUGjig(&edtnRHM*_JuU|Jc^0+|Rbk-*ePfX_2CxybXk zz_Q@}jyT}1~WQ$Ik53i?CeN`9X;< z-?Cz0mz&eoBsPIoGr{jwaqZDbM6NBmQ|+=kwwD*P^{Dr#+&?S+q0HBbF!;nEd2jb67& z0-Uc~&Yeqgt-f(iRVSj?980=&YpsrLH0SarXC7issxBzDv?#bLvR$Pr@KBy;IkZYv z6+`M&4o%hK*~!)96SB@P)jL@DZM2GQ)?1oC<;8k9Kh}&C}Wy97~qyVh0ZU8l$HEs_(8v#3Ickn6(9XC1BHyFOP2BCw% z;{+E5qdYVT+vw8|U4?b*7sm-l;k8#XxqB3%{G4Fz6fwA=sgi-JC-O84SDm^8b^`!=vq-GS>BW8l?%D`~SucJ5-NDMD1 z);SvjUHiv24m}v(#bE zQR#g_XXW+OxFK4E=*|W{*>3C*UkMu~c-#~gZzBMaO^G36GAWjW^>Y!#v~S<+^D;|R z)0;ELNq`kc13wwN^sTLrq|pa$kEyGmwmw}tsiPP)0e==T&3|sSPMb)~xi`2T1%rR( zirPIi3U^6Z{}eQ&UDPR>QTvf~FjxIGB+84w_3ip-9B*6k8T5hPR>~auO3dU~D#ZNV zmvC&5BvcqclJ=KsOH-P#Yu*@p$hL!kElP-QTvlNTs>c<>%9Cj3}G2(YEipC;y zpBma3o2~n|$LGMwpuc_x-mr?57yBEkc61DXP>bvj#UW_$Z*B2ow_oVGpO`%aD(qP- z;ccAOXm~<3OX_X>hsn(p74QS|t|cf%l@V3S?nyjugOmcw7)M=()-T@nZ)K6itrk>MN^GM-ZGm=$n}oVd^74Kxi6S_pK$8L9h(4op9X-Q9fA~ z&!txKSe$|8Uzvd;H-j2g+upZ@kC=b|f(6e{4FqW{&f2jJr&uZTsw^;Klz1z zyZqdQ%xLLqcFp${wMlg{(qXnIL?db}q+nJEB~eTueUUr)@TJ6b(3@cwQ6Lw7s(AEH zMq}sjAZmvY53yarc1vc2FfAJpv8#W|}w2P^2PHHC*|Xm@c~YOyP5R@~yiDwcp{Cf(maN~HFdwzMy( za_n3z)QUD>^lCZL7ki*`bk*xV*sN(9(s`)WeyFsSOhxa;{AD%Q#X47IVazO1CtD>{ zc4F@NmWJB>9U-A%)s1;BZriQ%U^7=GS%=l#BylaK<7hut)v^_g*3D~sLhdpr*nHf9 z?PI~I1LEj2*ic%WBLXFbvOcQXPr%ff1}y+r{anPSS{lQG52} z$jb3(tq6a1B724B7llKQuo_UnN7;tiKXJ!?&7P*J7EF1LQu+u4hb+i;X~coi&?*uw zwqhcL^24ea2V zIPw<~1Uv)n4m_)~fV`QeIiNb00S2~ER-Q6~Ni=~^`lUZtSHbE+gczx8k6n4C>18cl zh0^V&=dy)CJA9@AAHr>?xEdGQ zq;xiBqk&8#RPAc_lxiAZN{gUIXd~oIE}khtUVpH3XUmf_L$bZw^KX{^`mvG)z;@GAA-`kQhQzv?0MjP~&-qx%s>Fj^$0CAcd z4GA|tx2}6$_O+Ehm#+kETh{Gw8b7!qCZil}C=_xL2)Ubty?Y*2l!mXT(UUV63T_>W zP+37?{&xCdhhkLTJbgT(_$L`dgp70;4i0Ia!c)u!|F<`$jj4LAO;8Sur0+)gk*JC3 zqRmI3trN6WlECJiJfxD68bkJP(8@pR>n_yZ)@h)iCc9vEu(CvR zc}wuE03^Lcm;dwPr!LF#59s5`kDv>UgLNZbfezoq7Fr#5i-twVh%Uy?*a7p50(dH|}DM*gm~+Ip_(P z2e)E_dS*uY^-wR=Un<6C-gf!B6To)TOplQZGAnR^C+N4;Ux1n5T;@zeY=)jkS|Raq zCOrO3DDSR0x)oy7Pg;i~9+gsc5e+SsDZ57Gu<-I~c_Iry_L7?-`W@3RW}7|NK5rx! zm>j%b(qjPO5EsGPD2^hp31E_fNKDPYlwL^-m@JG(I)1i<(D#h_r#tMMZK%P|X5$8Id)OWB1r zX_^dVagwOZqgA)5FwW4x#ZuqUH$wfZ`6v9QvD!(4^axfbzXK2$M=(OL)+Si@{E7WM zE!utEJ6iyF|AI#cZbbSM9rUIyT>1is6Nw968uX_;!fehOo3k<@pA#UjQHhjz8Z!r? zSHEmZ?by_)nh#Q@>Pm$#F(PnR{(n}eOT<2Vs(ss_#V+ZnE5omBAayp_`rX6`A-|*48&YK^PS_lBxHVVm3cGisF&(~`z+XB6BWq5 ze{sy+wC4{+Ju|}umGP3-)CBMQykZg@)0w^qW9nXO z{1OBY_9Om)fjU+(EU=|~BSeTaHpLe8YUX3}>rMZd1Esc{VS4+1*x#6@YzeN1+GUAJ zqg8V=*Jx=)QmMXX4uI2Z6fy|~uv?G6X5k)O>}hASHFQD$dy!I=w+t;KKg~?(T^GoI|Et|bN-N9WFa6+k~~|TP6A^cCU^}FLjg{; z4tPM4VC2kL1m#N8PiPd3l?c{8>RHI+ZOn(NF&0CMH%EBWrb2SD)dfFORl~_*8T@TL zOU`MVYygjLSX0^i_lalO7S*(tFKCwkClBk%rA*E?0YJjfJ!--VXD9x?b+<52bd^6r zzxk3%9E&Px~Z}&j=)8$%%bKfWl0Lr)54GMgnR~*>)JRF!P8I^(-f2f)xVXtE;caC2lx5)VAssN12c)o} zOXS-u3sh4uosEA++5DtsqD(Vt3ZxZ}lqwaMDmA?ExZ83M`>wUrPSA2V`v2w~j_|b@ zt%Ozu{IdGHMr8o#=*Wo)z*Qs=u{i=MhIv8IrVmPnB~sEGBUe!qmGhe<|6!JRUD`#} z$swhO6k&}X20{%7))35YFaUxkC8fRrvqldQG<&M0-j@KNPxtT>N*k&%z}0ynYQOmm zA$rlZD_S0EO&o*mW~F^d_>0$*d-6?Kz;DIE8?aK6RD(M`^l4D(vS;TGjYtr{+*+S-NrO`Of2fonDf=J zr0J;$HS~YjdZ#GOg0)RBZQHh0Y1^)}ZQHh0X;#{{ZQEw0t#9(2{%6*7uU`9N-|rPW z;*E#bxn?;aj3lB(2mM6{KYVsc4OhX?ux#XM$5=Xj#k}L{yb=?HjJzf|Q4REeNNnK6 zqkN{_Wfo4CI_m@;(c(`9l0YiEpTj^O9JD;5oUDb+%4sjS>9<+HB|4Ul>Vc%&FdL2x zOvJp0)0oHo-Et>~{am_mWldlzUBSHZ;Tdrlp`i4BlYt6Pqrz7c|AYai1~D$wU39>) z%nh$&qQ=DnRU=cU>4N2v%8(Z{raow?&8?-Pr+H%q?hN%o~(o9xCKS zfV_7HPD~+^wpu1B>Kqy5RVp6S6bT6|uWlUqjf>fTEQjeNl}vH&&8>3541LDxJFQ@G zM{TXavGH{r@%KNvRW`03k?NWE53(1j2_ok9cVjYMdo&a?i@Aak9AM^7-^!s4JR&U2 z1viQ@@-L3?~Ojlc$+=4X%6}zuU_TXzZV`dM-f>o%gW#V znW=Gf**Yv-=FbY}O^46!g^QxySnxT!x_Z7pVF?uYSvhOY4W-lO0vg^xv3@udP#4lsrPNAz@nwu$z&6^}VsjPj=3z+2@R0!t0?wgr}E4O3SV4^fv2MoLj4Qkon1i zNuTn*M$K&KoGff)NT8jhh4Tjva|hzv#xU&)#iEtrSnJ9-zu4n(tPld^^1*pU1EtP_ zfSekvUL^}ge;|^+pD-BAN5g{K%CY(~{r|<5_Zjhr)PT#LfuO#_0Z;WKWeg=^_vl8Y zOX!5%%;7lDv>k9yZbDbVi1T}ZYvxRm89-bk<$wUY7mFu~(kR&}T1<)F;DdTT3xvAC zbPoLc;M!vl?PT|WmY)D(=xob^aRPU-7~Lc`-nJhjc6rDBOrEpm6u+wWs5P@oJ!rsr zVUPE=q6i=@0r4TqHg?7>@ToWqc@%x<35ANM}4n8>Z_-4d@%o82N?Coh#N%~LQ z5~ZQk@6?3c0k`7HItRyJxZd|e7H3>0XP&|9|2h#L-Vqy%+4gJ*9ER`$+zI>i`R|cA zgZ$rgpVhS`waRl=+X@YYJa8Tt&s;Y2yL%Jm@Z(W`LJw4t7m+usftN&K(^WkY7Y_;G z+%xdWDp;8kPi14h$;yP(@a=c4NGn7{tQBEg*U`6^CU0TS=78MCfJ(c@R^SVbguo?; zPS(g|{wxW$I_ATc05knU^c-?3NS}+P;0PGIeCTx%QQVtNhoyOOK{wyJoxb+&;IEnQjwGa z#L55+ASzg{c@i8&QL*E!O}Bp_gMjs0gyBdyH4r2bF;!2JAl~IjA24cOMKf{50%D#? zLCn8Bzu$m1_BJc;2j8*f?aPj4ResV-zoC7f?y|nTs9dI@3w6WW(kJN%NAb9F;kk@P z3CIx+P>&joY0Qdv&fsVMQk_F!8a|fH)tfiZ|L8t`yj}MIzuvqrz~`sS?eyN&!J8-G z>h0~}o8v3FanTQeJGVOo^co}bF6V#V(lGSv2sy_0^J-imG1^2C2TSQ7pmdze4`#$@ zyY$en>`Nm$ju=C%kfAR#>I|F8ZKj#*pl?)E`hs5qNKF-I9iPMdr52(l@) zmM1#CN>>Sq!;jcL+>3@g*BvSF^yUpb`Il@ynvXk{<{z=jFT^eW2|zMUZ`-@4QAYQ! zEr1MP8^I%8`5^R<2>Rm%7@Gemc%52)+(%G~qEDGy&bJOE4a?>c)kl>u6SwN_LaFRr zy?NFK-?lv60o;k*{5&sWcblODTYn|Xc*wb z3`n_UCn|->%%~kZDcvI}t;>M_5qn551#>S8$_y9kebuW2TV1p+0IaWYX&=+&P8d*@FIxgX$Q$5Ts@O_8CCd7|IA%IvJlW$FfT_Xi*7yfXu4 zNsNXK8E;em)K3kg&2X$l{T8|=KAF#pA+^8^uHp?*>@t(?7kwml?6_5ic)~4=Kh11K z%xHbE(zgv8f6!JmR}D66)HP`wX@5NR!ng%sws}-?fK+?|%)gTREk^+U-(9s|09;S$ zPXPE&jv+3v<2j;GwHK;^00;0rE50ps2pqZ%INh-y;-!^X^XNWyd55`rwo7ZPyKPp1 z0pG<}7~i%%Rr=*H$VQE`^{x8U>ud^&7Adga@AEHs3)*Du$)OhXB`*DRZ6?zf)Ieu~ zpL?OHvMMvN;9UeB(iMn`ZQW+%SThq9{WmrL?9s_i)L+3z<;Q8|)AT5|sMpS0T3pyH zntr3mx|Rl|x=L|kU!>=Vw;3E=O2@_=i@5=^W9Djc$tq$myCFV~rHoCJu<+4QHoXbC zhB~V!XE+c2-j<$k{xhEcIH0hu*vo12m3Bxsd1ztStCDo(WQW;;b9H0b=`jAnKgp<~ zHWq_Uc$U0k9-Wvt`Y52S0`;N-`7@fC^djjcu`v#*#}CRIcU`?#2z~odGV@DZ+_TMu z-a)@L6B6Z7NEUU8d(M|vz}6?GUt0Ot2@i=PiU4g1J=Hh!(zrG?B`S-&dLmj0Gr1*7I8Zan|5M z8)*77>janJQSxGD^vi-hnRBo~i5r$rg0Az5s@Kg?2_-JxR>FD*|H4ZYCI6^lG&K-$ zKjmtisSDfr+B^Ox)(%%gYia$R(-d@=Q2~2+!_oHd3I%pO$@+x-#&`gR;fgv4%aa63 z3qvdhyJ9|7=zS3gnltAtzvl((GEr z_0e3b1)yo@uRIi?Fin1ASq3YTP0>aNEFcC-j3IRYWP#}N@SFllXlBzj5yt>-tFE5E zTLWur)y#f75h-oNmSjn$^8GB3IG*^#oFW(fuhEQVer<)Q*^-0*7_AFBRvG(FZXlM^W z_HT|7CA^yrPreW_e_i{4BUI>fisbVPTCqNebBKklfPuje5->>h3mEHjJ>+S8mQDS+-enI+`m>u&I zUe45J)*`cYjrDFk)FHeYLB1l@%2>2QH$>cm>KCtk+FH0?M|>DoxMt8}NirhtXOzVO zAL{HeSg8v+&u#ZSVwV&XV&C^aG9SD(*wQ|Et}go>A5qe%wWI0|I$r|3$+TsM6@d0d?u2RxWr5!m2MS1sC_q@xvyhkW}UA#&8q->VY*?`NGqCbIVyKEx>1 zWF*ksnGN%3y?m2BM2buY6M#*)H5P8w4azgJw7wC55`VHHjXIlGR_&uw8Wbq z*--m1_3ga7=_s&UU@R8skK)8(!~FtFef-}n{;!%O>Nn5mz)}cf%Mhpn;t*$1GkFyJG(qSEQS$Owj!1NSGe*tA>8oR@FH|H^bbClqDGN*y zBdn}R*+5X~;j+XYDuap@CS-;@-Y68Yal;b&X#*pN8?mfM_o?jo{$3D2MK4wC0obKZ zR*^Yor0%*Xzd{HxLV77be>v)6L%&AS!EK>0TeR7QU1GALU!ckti4wSH24(*S$19b4 zER&MI5INV`-d>fPVaqRpn5|=C6(csGZ{K`7q_?+JPuA-0?IW`nJF3sW)@J&es zyP{V;ojnyC^Ra}FnFbO!Y|@MqQDoT)0X zN{v;n`fV3tG~(l_Cy|1Z&D#{|iM#d>!>qOOMMkVr$OK+{J?{Mc?@XAbAH5i-GrJ|h z6k1Zz{PFFa7(`H(@-=+~(;E)k#1;8O`n;$;j^#nxsaRH>2^hyOA&PmUadi62;Ec~8 zE8Zovq`&(ob$cg#`!`$6Mn=DO1VM0c}K~tcD(UX>~1>}b7SbOp+{qg(4B7{Y{?2~?ESYLF} zPc5B){m~;4Eo*wd&O#y9V^gqEjtMf9V%3LwXNvOOiE?4z;}`YB!g&hPY}dbfzd5@4 zW>EG}(E8xqn9~tSirgCt-Q(^y)hu{x!x^%S@wdc2m9gPS!h((@4Ptiv7aN$bK zb2uX80}aGwXn|l5#3G^V{$XTcdgQ7TPOAe#%rJUlB5WRLj8TVqKyC&`2ZlEkbF?=< za)lRUS*wOM*p9ehF7#3ZzHNjb2_j#!F`%K41YI-)UWCl^$MVg`D2ygnQSyYBg9MeU zXM#*n_MyDlz2zbzO%*YFiV*dY7jL-EOdyjqv1B6Qze!T!7;|bwBS6l>H8B~hUOG{(j~wy}X#mHvvJO-K!v zYL!SL7}|!(3dvMiZpF-`N~i46f40TF za)A$80h?4|m_!(jUUfxOM&j*MLRS8mlFK+wbo+zvMBIgF<9Gf?oK`*D8n|fA*$EV{ zn{n_&)z65(r_u=@q-)NbdkgMWt1J}4R<>A*Ok7!9n6rd0Z^miuBSsvd%LYh#0?Siq`=`-eVjIYjrY|>@UHG za0#jpw(kPd0|x!v8}Nu!{aY92B%WN|0KU5Xx@q)~@PtM@EgRzZZGdY{Kx0s#PNLtv z(_I`C#;znrHOpNu$A(fGP4^#H1MVby#+J}js#4bivoqoIt_ew?R>4A$R_XcfK$aOC z0}lQz*Xt_af4-eNBdBUk2JUp;7)hkz+~7-MlpQ9o9y&&*l;M});$*AxgMnmnku7zI zaD0A)F^2ARegG~#|A#$EM|d2i%wQ@6v#Q(ob!9Z@>uedxtLDL|qBYs3I42u57$%`| zygr~w36_(7ikD%uI7#1}2NA)FlD7m)2o>l zuf?OsX-0r;Ho3t7u2vO_)r1Vzl@ zNT#=i+76NlPzSP$blgfPWooX?ggu4$NW5Xc|& zWOZ4C7_WAOaJ3Wj5f3l04SG)3u{5pQz*vc#E;5VI9;QVr>t2MwpA0Q7GApG&cJhq% zY3qAP#w{rV5b0q+s={^SldsGsFQIOds#ROQY_wKcm^Z`PzEOY0TM z-|j#=LN>6^K>Nk7Ahk^9Lexib_0dOf#bYKo*`Sb}{Q43}uRjEe$)wwqL9tRGnviqQ zcKuZ?C}G_p4>LZTQijePSxV zz~08^2Xu?l^SzH}&i;f!^Iq8gFD#8H0od)1F~_kB)N z@v?{#G&KqYZJsrFdq`l3@$V)T%B)KJFZFGWnWW2~;DP5Ge*lkj!)u$Hy(&h!>vJ)0 z`IhA`$yFq=>9A`rFe{bj{fHWVKWB^o<)6Al`a$&OF49Ry8;K>PEdA+bpWPt4``7RI z+~}QVy%_2h75sLyghv_di*|2b7-wdV0AGFr4}kOGhlk|-YzkcuJCC%OEz;?Eg-rihF4F< zNZvb)xb`oW+5bLF@_0R4&^-potuq&1?vdW(G>9mk58D3-b*fLYOqM1CPHYZ@)M)ca z&jOPDKQ1V=M~cLgA+or`&UA%%$}zW&6ENkOI4-fXwZJt#gdQY5IWq}Eyv){7REr&c zPdB)Zz?9c;54m<40HRV@BrHQMNq6@Pj+aA(?G&?7H{dRu%A@>60k4}ds#iyj@EWOj zO&IzKs{@16XslgCRWDo$+2TJiSl#~}2IG_zME%BK8nmBpdQPB)&t^HEkAsCa$`22I zW)Paf`&Id(prB+Rv55822+N06+dwP>(F@ZUIiZa|N5q2yh(Rk#MXbD z`9id_YxK9^4i}viZJeou_0EU!;;+C>9rU4ii4^3Uh{wj7!SkE+cR;wtg zx35rUb7eRdQbceLw>D|1rV?}fgCQ__Q>!|oDKeXhT;5F^ZnP|9jn%9_BZ5h>OOo*x zyau_zvO)^5Ns$3Vhv%UX4!Hp*xI}z>fHd}FJT}Z^%gt;ah{Sk% z8-*4(P*AV=VkascS>&en#_te4$#afxBZmAIG{9eKKcEhv9R%4%oAOu z-WTeUE4ZcdP9Fn(V@j&3y6>rzOv!fm#GeQ#g)o3|s`p!?Z&At>NTJXOL0jHupkgX8 zIoX47ig6H_b(WrI` zyP1VvMHD8C#tXnWSwq`GLj9|SFQ511>1j^AT%c1Nh~8^;SzkDY>8Khfx=bognWdMy zhLP%_RO9}tUIF)T z=q!=iE#Ljn$jw~@*=5={2mCK*I-n=7ZQBRjz|&L{d<F zz8MTh2lzMp@;{Qbe_~AGd4!J0AvNzev=^#|kU`J3h-d7moRE4XY7&MK z{@U?&Y7&@g@wERdkjL3ZEZ+h4@mlu*`Wv?ZuYJINpRV)R zy!Q~bo5@V!{AEzvbB)iWX!Etsa^vAao{?UlLw}Pcc+XuE$iD34-Ru|*QrHJ_p2!s` z4p;rr@MmBdom)S^1D-plb`MhUb0k%*G6xjtx@5`rkRbE+k6|Pz64DD1$%^mKht-D( zYKM-nenR*_MJcA7)VkI0HFI|aFfZLh0X)nPyCb>t<~*jahCi@o3@T6fseidYbtLcC z!O2hNV7FG@o>vc)HDtHrcHDC1wpKd)Q7gD>Oo3G@JN9rj(C$RYyXS5)Z`2Lz?0^G} zx*?X}Kn&S2^!FN6?4)pamou9->lAdi4SbJ?hGrWDC7%i2{QZ`qZu)cI_7>g@Z4ZV` zRyk`_^tM$$nW>d!E$GtC=4s&MIJvc5O#R!K;B_$iiCq~^B95^n#Bt(Q%6Kw;8K*qJ z11T0gc|Yl=Q{za)jrsH|N0D?CAyda>pYyM}HYb4>WMO!*uIbNDSxsW)+3iH1^nC)Q zi6AC3qL3EjuB*Y>f54N9CSR9uKB+-Vm-|UmNk%u3_}D5rR$b?@VbaCrzn?TflFQBg zf%hmgV{2FNQDaFXzgV;&alFvm>U!hrl%jpKqy3ul1TY;dh`{uGHxsD*^8W}vOg;<& z-0Z!cH9Y~ghm1WlalGQp_>RvkeWSj+5E2#-SG%xn~FdI@-*$^DGYc_q%KG`^HA9@cO_Ept^ly9Fe%$+tH=5 z8OQ@y!KhWY1e-IDD?{vd|KkN#g-I^a2T?zw;{nzoAdL7f&jxhoE>@HfridG8ZK>0a z;_aL!KKxCiRu)kGa);nnzUSDy37k*cQ4TcDu}%|w9UrguYxoZ^NDizBe$FeJ|6d1| z{u%PSx<)#jttn~fAM^z3xIR72CLj8@^wK-f=~zs_HST+3n>}GfGz0W8zRKz*tZ;k;)r1~XErDm9L{)o5J%n>q7Y7^*xx%$%OFZ} znV}dY!1-Nxax1VIuT0%Nn8t{W>yQJrNG^X)YiMu9s~pOYmQ)n6?-NMbP`xi$^ZTjq z-@FeOD??ts;cDDqs zM}x{J)$E~@&Qb482NnEqI&Uj4Cs`ge*8l-&agSKtBSEbdEX6v205`YXH6T8m(){1g zCZ=HfQHv6}Dt@!I2cg%z<;v_l##cD`&;C#ai*MwahkuWpV?QI+$_8-Yu(FnL3Q7Yv ztlnUl8svtLiDdeGau-#O?{&RHU)2BP<`DU`bi~hqz}+58tCmx1{qHXdY08=c`W1dd zSnCPGLC2I@!vs)N?1t$-A-3jK+_7HDd>bz&F-HC>oVi!Cg1;z$bS>{nN;Ce;Gk^PE z$y$s6b%4hjA;p7#PEOP5595+K4-l4K3nH)(H4V1fE2^~L_gGdsOiEq-eJboBSMHht zl%;Ybd@DX($$Qp3%QP{dZHz+j;Zr1E8Jq1^l|BgZ>ofwDFB1ZvT}eARPN=LLN}RMU z%q%B8>?vgcNkb*m4a$K&mIu5z1Gu&}P+%s%702^q$%EzsFy}ZnPZi;EIY;h#al<8?XcL@XWrPh(hKe7xZ*F73Cx$7l3BSw8A!yXjR$ zVYw>2vjVgJs+yXKfJ-B}dqBTI!aaZw=Xvv?HPUA& z=TAbyQJ_z^sce>_nM*Fd5F*4@f14p<7~q$9*Og=BQneDMxtYLqG;{Sw!(jJa7!xRp z7c3+5?zEfssgklaJ}tQ^?mmV~*PT;0KTeK))iMM;A8FT)*WJhQz$0tu`S6u!7ar<4 zR?29Zpf%gsjJew&kJJm1Pi6j-;oLvBD))AfCKkfkjA}sdi{=u9uu=;X_?r2+5-k&Q zDq4w7x)CHl;#pyJ*e-@hOqGUOz>x9E=D>9vcoI0xr98x^M5q3$6BdCL(>Dl9U}wqo4>Q;nOjcta9)>C04MxKoYp_?lcfmFD5|%Z*dx4H8+)mqc>csXQF~^GA3qo0N!oz25sD0`9yiVwWK}&?V=2p zVFBzqXnJ{&e|+?U4lA<+(P~krm|U)rnryTO+vH8V70;7{v*EdETF&WDHS%Y><`S z%7Y9rmGfHBSz8hr(x2~g})}50voneYb3|a zo4M1>yq?-@LiPSwu~?(QcsjH*Y*sa*snpE1>Z}!;KYfhH-tzH2UG**x2u6JZ=9RTgJkC~pKS-AeF#Pwvbo4NTM*2No3U{n z#g`P(W$sisATh(KQd~rQHV zu(kr{07qVnF97@f(la1}ZT$7#pq)RU*I?v@Ny51#$>vzgM?DB*s%*(lxax5VKNLe` zzebohQK7@$G3OUM%q%I^c-Qc9%Iaxzj%2RH7ImtE2GiVCx8NAOO5Q_(X43E#%0Sq} zKeUFW-1Kj2VvL^YzOl!Dh%9Lu5_MC!zb40TxM(EJO_U(`(XWt?6o6CxeF6 zK#?@GoK~e7ZF^&dV$c~yC)rStUX0CnA`$CLl2WBx)2t7smx!t{)XSX2Y|SiS-a3R! zP!g60P*5mmXhfhIU;u%cEb3bGN0c2>Gowz{&GlCyWf9&w^K znEa!;*lTdQR5F*E)z?ecFMk;KxOQT4RR{c5x-$Q+ie-Z=?jxYgUL}O1|9-&HDV8X9 zkTNL%C(8I&(&i%#I&N1^#p&Uy(GZvgC0Da0&JD%Y3ajaRuhcMpHD_GzeiBAVDR=~d zeY%GdhMR>D3d?+X*3vi3vBLLg?)IWQU+mSr!CVpOlfqGM zn28b**ydttgN4FbKB&Tz(li;*iA~c3?oM<#pl!RXp<;#pgc*CkHc5%R&7AwvlCjJ0LF%j5n&Rn};?Oi6()tK+o|jEN3sCqN zeE#$&o&CcYE)bBH$Vh<_$ZWDUABB~U1j9ms`{_`oo)%59##kRi%FWFEhxzai-%}cl z?=h+V;+5W_Qd=dFdDwwCdM=JcMb~!$&rgnr{R?ocdB|41Y??t2%!*>$cR2J1V1E9# zN%!Z(si^g5ZM41(OyFEOgj6t*%i7^>Z`M7rH2UhKx*c+)aOt>( z7kbl4f8(>tpDE{N)IjsYGu{82y`!Xu%ycY}kezPEd{;j@Og%O4U{Tj(Maa*ntnC6V zv$RaHRMfwT@kjTIA0XEIRd)q)VD=7Q1-2u-%=}q!$F62FbN2NPGKD(@a%v1#-@X5MUo}hJrYKDarMOeBiPRzv=Y>0R zyeWG`dfgNtmX5{wqf&9F_`eyo1yWJ?jTemdpCHf^!e{U_#U#uNsO=vZjwJ3b%L8gx z_o)j#u`&PJvrO5icvta^`n~AgWw@Rxu}$zj6MU@8(mFEL&O!YOL0?a|e&Je4t9P2> zMARtwqzb;O{I2w~LlxWdng*9bcBzlY>ilaG)39pRCELXcGMtA@NPeq+CmPNQ#>fBR zes7GbV#4$VJ62vOGndG^pal5$x75}YX+sMh@O3ykc^UU^UtxG<99QrH@pELn>y2o! zrUCOu3I$M6T$488gJ#<_Fa1tw&Xn7g!Ol7fJtP|8pP@2hJ!qF}$qvb}@dl}H%|BDx zkRM>!zi+HL3A|APwvIrzS94gGEXliKcAV=G&E=(l;iuKAl9{C#p26lw zt`RG1?pN-SulR|0E-KEv{m{2s>rG@ z!5JjweZFyuT9X&C0ZHV8UQ~FHpZhCoOS52O-Fjyl4!X!P zR`i=3@v@@$r546d919Te3R>VkpRUPhY(4dq!R&6y_>jkB`l+8j`(0DPCcn$?*l2`a z-HKk;g&bpWMdHKta)R**Z#)o588wXPU-bul6;R{(ZXy8x0RX$^#K+Ru18{Ikp8&Wo z^6;9!1#mrA1XO&3v30?Wz`xoYN+U`KZa~IC5pN)s!$)+0c+&xcMx zadFD{BiO82We;;haTbMm>(1duJdi_xmO}M?yEqs9QY^XAsP$njMeT#+&Na((tA=-= z`B08zame;)`BoETF8c}pxs)B~&RyC{ET;2rTTc$| zorblR(7KiEzE5v--f^xCMz`0>Go}`IS6<(%A8Pz7xw$G001)Ei=a9Vu&JEfw-}17M zs{9iiIRdsael0FnmEr+b9}Cz?h&lY*wRB_EGd*r|73Y+|O5DW8;ozc6ui5n7$Q6YKldh0}^WRvk(d{c!e6^3qxpRV9KnaKJ1&Akt#qM^A4O*Zn6#qm!Q?Zno_5F zv>5UHo_tYlw2rmLQ{H)EUukm#rnL8E=msv-PCdFE0D{HGnMX(9s#aD%9$x_b#ahy_ zCwK+4Vz%gX}IQcpoa;ZKT2AnA{drbR7D(LCP-? zarAwQOal$D7=f@{4KE^$lMrHKR8fO6*f{i{UnxWXu7dxBP+%tX82mEmpCHGm+pBIC zG;w2PmbzfzNER}E+?aqiCH;sW#yS`7&U4h*X-WxtyyPoH>m1uaVPWf)cnq2Xl|(-K zq{MoJpzQW*-5nng25PiqzdJCKAJTAMNUeY7J)^^xhkqGJ%$yD zGv723`T1znpWtU}rTr(^Jqsq(R>2BW;ki^!!{i;e)<^wRXZ{j-T zq{Q-x9d{%0WX;rKO+;7hI!4otqn5p0MeOHb%^dPU^>PE9?@(otj$osJ277}0wrMF^ zItKZGIA2^g>K`Iy@ehsJMXsS9llM_8Ybh+^vRAp-Dl>fu(a34EZbm|XBS^*zvm0V$ z#6Cj4rrTmUH&r0d}WQ%{#{*pr&}zmnuNcjXvyQ*(XAvHW#| zgo+0hg};y9owzQMprb$#xuIOXH*W%FglLM&EZ9_K^;Q>*4*FE>1fd(@+ETN2EDkDQ zDZGpB^-bW^%+s5+VT)C{+jH}h>+cg6%xDg1#?qw{*A5*-`g-35%Hi_7ODda{ztN{; z=uUO6aQqQJqGxsQx~)M37F$sRo10=yaJoK+?8-egCt1tfYC7RS>RA_$WAlEOoD<8M zOjZW?`DVPIc``R{liKr|9$o1dTM>l8n5{iOmDNB zetRB0mPv7nT3}{;6c)c!>9c zIrNv8M}5vaY-Iobpne0eV*%*NfS!ID|2|@YS4o4vEGIfWBpQKiSLmTtp}}rUM1$Nn zbOe{Db$}f=*#s3RK)PUi%E;zGF5@TrLb|(C$)zg^;8{;Z?Q3WC6|dd~g49T;;Zf$M)!=BfY^eL--|1%ls%@bU4~v{W7LP9+sDc*z+UcnkdhNQc4kso-Wg-K6{tP<-5bThgWF#Kdl(f`0`Pc0#Opu zBbMU9?N5!vtivyBA}U95ePCX2mXxURyg(8b;ui<*p6Ae)|Hv7FZh|mr9LK$}x|rM4Na z!y?A^MbiR5Z|V0WtS)>8eGQv0_DK|&(Kvx8Ff&^uW!^SAlY8XmVcLxCVpC(ag_msZbnQXMOuo8M?K?jg z)@2&)al%!e#q%QwTue?bO6CKikOd#NZ;*@1|ERp7uZ7wdFhr~;KeVtjdo3qO_c2yz zeAg$r%kqPH?Cl@dy#HSPzRO?!x#aWOK1`}i0q!{?cUxRZ7R_R)lR0aO=)|zk_ft+( z()XsL^8K`gt3AK-`0ECapF95iixXsVJpY{B$9}W=EI;d5Hv(bcGlEGUsgR{=t z_R0L&#`bXL94o;6t7=}=CPiKHJfnJGEd$Xw_d z=~KPwLb_3=Opyn}Q9ZwbxqMUdeCE*&RDcZp#6~VwuhRt*#W>a{I6a{t3Wzu8*CRqL4Oc&*N)s^)gQmFss zE`1NU@SbE^RIe(7vF{cp2!easVb&FjU=;P06w$z+dxJKr1x@1M{H-hPD63A?>=t2e zi*~4G$sT5<`rObph(bPGSL_SC++OLcEwzA99k1$4EmjRSXojawLkWV8w>>CsNz7Hn0X<3kHRxNiy9Q1O-bGCUJ=JTTebN0t1juu6@qdCr{ z)c1ocUlL);q;XUprPDR-(ipPX=5r(_l`lfJ4WqyKAwGx9mh_~`D zh^GlJkZME3B1PQ(&l=?rY=P1Kt-I1hxc5&0{jQ1TlKSZ$WpvK3&QJ6Cx`5 z6DGiP1V}|fXv0&gWIY5fx^#c(9#`;2f|#pw}LdRi;~ z1E$BDz8R!YwGShlW>8Ju!)b>9&91VRwVt$60P-<=H%^oYaP;8i{rOGU@7YY>&cyj$ zt;=K6>%j|urUPPrP|S%90U zAl96$y7}6+L8obKZk*T?KqX#n-ly40;1U>D-8IYYSgG4)AYQ<|i-pf^!cW|d&-DjQvtE^Ivx|hC1%CyXOPO(Z)S>b(ue6m`S zeGrCOTx4%z8TCrXTr2K_L+{YPha~4D{OcBYB@K&0+_QhqP0iJr;cmLh3OCP8&Gq${ zyJaso4}D|9YU)y=Nps3DcgAdE;7M^uV@&>Z)nLeiH3KVtYV*OL9;{Swf|DOmb9uSU zKcjylf(&WR#hnGIrsF%FuPc3YVFcqJXw*UPEJWnP?*6%f!^Q<5%Y!`^@Oxf?65X?S zN5+tqZ9u$-W+Fms_N-O`afQ#fE}rNzhX|weQMd;e>FQyiDJ#}wvMO$>!iB*qlUFY-eLoXKzY$e3Ya} zumV>|XiAmjP3Ku*m5+9=TS)VKoY6$7DCxy&q7)mr<+<#?d~<})0EDDC5Gm4yeCE^D zb~V(gVz4)MrF>E}V zJ+YbxpNob&qx(8>2=YDK_c!-9!MhiG51v>r-;wYYI>(ectGCCvJ>$zJ$usYrf-LFP z26}^g7v<)jdyFQ_U*mv?C$jtGAU|AE#Cl0%GF8XydfgZEQZ_`Y zd&Gmuo-^5JPJOF{ikKou11j)h4i7w!kRnxs)DYYf{f)@V08k|UJnqAaR=kpMa*AQk zv`{4Hfkn-U6u=C${;moNl`YL&%!16o?ZHdGef)_4<7S%YlXl(uTZ!128*TxOMShm%Ro+mfC*b#PY&>Z%8@3n>-Lk z*DHS1-r)BOALMtsUl=>0)=TCqs|%kF=YRu8hs*e1hndf=T{+t+ zP&-=WbP3U8@T*@tU9-Hjr7xoW%+5Rsv3-Ky9iu_K-*LkPmT$1Rhr*rCYWSht*3U*w zRfFjNcxVfF5rdnKK%z5xdsEt$ki@a^CDegS#fh|)0 z>Zcaf1+?|57G0E$Onhv?X7MTPB4~v7pe8suCi#hFPEBTagQ1~%dUiu*xcEdQ$okS~ zp|#hs@TN>2>#9clZDdTND7qqx87~W8+E?&!M`}d0kcJIQr>l$ne#2sMDGJX0BaA0r z$hgom0Aji83N5-0Uz;G2AAObzP7f^L#H7V#eX=YDSA)~32UV3NK2$a3szhZcl@YvZ zDIKFfL|Hm=e7QQfEHfG-D40$JqGYUE0uip#`6J?&^t|DP zY8nCU@%!4ffe5E8EzU2OcSRXQ7U1)p@Rc(p5SVM`jD#Nemtx5YMN763;DQ4iMlZsz zb-t|q_()uD$gW#N;Qplc`DCSv>hRA3g_+Q7%?a(nDnJNjd#ay-6^9_>Tl5U?z6$tu ztcfR4AND;L))c?%2Lz=eRh1|>{(rdo$Kc4m_xm4?ZQC{{$;7rb@q`oGwrx&q+qUgY zY}-ly`~Cg=>V9xP?5eIlr%vy^&)(~Lt#w7j4gWG7D2q{3Uej)+mvJssMGhcA?n)sI zfz9&^5QKRaKFMoj4l0;5VtSf7ISSlis}O7zh|`yq_1k_eA{&M4>&C)s?c4CpAhc08 za=kBXyJTTqj|glC;qJjurT)#Y2w7Jx#CFh}7tO7_HvNeS*&v=|0NNl)a5@chPzycS zsNcJKgGjgF@^~I&q#!5~T9iCvIlIup!F&Oyo=5M|e7U79ql6Z6?^x~4W$6U+(bqfm zYyAkHpjzELnYflNr<~sia>GU@Ms_}3rKMACHAytJ&|fnzo;+H$D3oy)XXp&pW4oi#c%806No4ucwi{KGauh5A}P7> zvY2V?aWE6Hl|9B|hZO$+N{WyH-}gbO;@>Js0e6wF*tJ-K5&+UvY}s?}3{edCx%sRZ>9+C>U=s{xf5{sCw4+KKr> z%83fqgNczJ3+?L3>C4TD&cAx{H+<@C+<7*#=n_kBsN^C-HpI8gf9a>xUBGtdS52tT z^{c-xt6sS$vHLaOPD~*16D&lujnL`++a_mNjE2&zUc9_c(JjZ^sQ5>GH1~-v@wc;r zxZ;-^*~IW1Gn<}iU#(e_c1+K+qqpcWN~xMxIYtGTo@Ocy&TY!vd6Jw)wsPeMxlmQ|@o%?$mZ4B&#BkDTNUFTPGU8{qQY=GD8l|{K(ON0(c5g09Tq9yl1^* zIT*VUSuGB)Ulq4z#*It`Px*i9_Z4o`O8MvzV8>Vxh}%Uqv0nw3rlq-%ouZlJ`=#Ta zfYS5f(gn@_Kk=KId#sn3a>Bguf>4{t|AehRqp7FH5vd~%ygAktGW`%mD*e0?EEkS# zN__R+k=bR6I5ke89D0yjDl|KU&y-p~GS@@8Z;wECwDP3YNwI5~pqm``k-5=Xc5P$4TzKyxHslKW!iRpt0KkX(Wvb4^GcB z1h3^Vo1^%0>^sQP<=&SeadLaU=-TFEmBPbVuVLjs;~kg}s;L$;PtBwhf#^MvjCsp5 zq{V7~*UsUxDqAg+b~QS_f+JEGORq+*Gdk7*_bYqh*!}pH4j2! zG(ZiSq5mm^XMc73S-I8-_+tLaGet5;F43)t5c*m65tvimS z*7^FDU|EggpsPiK`shL_FJ!hh=)8~e?tH?9>!km*aEg69f@&8lD3O$HT$0(iIW1=Z zfMYq+AA`_E0t9#jWJdgufhW2he2jU2j}h58l;O3`rxZB4gR z8Wd>9>wNvbz_MgSM)GsM3)Q~B*7y62FDcYI@LE~u3a)ZyXK*ZQD$ayQ7R+4%k)PsQ zIB}A-j5m~C9e{cqa=o$_cBILydU@gZCnDuYJrnt#q7PnT%wv517y&bWfx>q73m0x5 zISg`W79xG6*pYqupFMx*!6MjB*(G}@WK?HP!4itq107?|#6*UrlDVYCi_$;-li<89 zEJJBcIxozFd1(Yzj{v5Q{up$71Evk;4?V~Jknp&Rb%2$Ni6-!nt zHNC&DuqVLvN8lrwy^sbma^nJHu7Ox{@_Q4ZB6x!YDqAH1aqLqZ5#VF#e7~_{#*;IJ zE~l^%K2+_AAWobWX!7D`mx}rR{}qWHnX0HB0~V&JO2f2Ro_vqxcu?n%eM*N!y&O31 z?zw&5(Qga!k4u(`;ek{qx1SMPme8syZ?9tO;#-MXJ)2gfm{B z9$qbgO^Ha*M#j^vybj(EYy4p9y;%arM*}+XhUp1Liaw=5JH56hNKbpWNT-J1$)g5>X2(R<l6?=z)hBhD zi*gdELKayS>%$)3tIImNPdx#bC%-a=+Ts>a@e{#8-}BGjop*rEak&|Q-V9=cEH?>L zpxuHc4CLa9kciRX?ckbh#i3j=+EpC+15=A@@$H8V^etV)51!!%DR?4ZKS_GtyQL#Q zhd25s;PsTk0>E>Y`#)mq7;wX>$8sqDxFby(@1eY5sKRkNv6H1$&hp32JH3!2D1){p z3facUN2c2I;p;08%9cz3X>&JB$QFLI`)hr%`-(UI#6WGY;-%(XOb z|5ipP1%Y<5&;NtQpyW+Y1#<#CNk~Jl$TQ@SZ0I{VMLzYfITa+erQ%TyjSc5i7!K3+ z;#WA9YJLE2WR5@*SIhjb?9h8Rj>V_5V~JAkw)QvvA{f*7cvFcH3a_Alx4hy3(1adX zI9WM?$CZUl8g}SyQJk8{AK7ZpDQ6)iv>_W}zlLYBA=wL3l1=Ym|H+_EEJpih&p~oE zU9R*-j951PGq3Wdy8yg|J6;Mglhmxaut9s4{l*9C?Zk+)iSK|jhu&_09eG?apqIhE zg>8{g&&1PR@bt1)ZqcOdD6#DXwb}fO0uzw29=MJ5$qv8p(!^WG`COnU@?58S82?7_ z##Ar`p&({QxAS3aMEz)yUS(VSp`4FB(oX-Wq)n=Lw)@1(gjYKB5buR~ycVJM@i?j+ zl1)*52|HXNjG91(W)J~+E@G7*aqyKNeGj%UMn?LAeWdoA}`lRqP7 zgW)D&(Xvr*(D7V)-I9bqcKGF*;#&6BRsVgjiX9+%vJJ9kM|Yp2k!$s+RSq!quEKez z&jtIsd9;q70I+i7wY&opsu4cyq~pFIRqI8n67cibIoiX|i}I)99_f|5sZuH>iXlW? zb9bF(<}n-}6HY&94?pa0{%*F{8Kt3zyudgE@im3)xU*mXmR#O4R_-2i-dtm?OXQsQ zQ1Tw524SD`!vHpLqZOlq+G#hv>2$O2OzwMzX5YzNx4mUC?Gpm@?5hn+Q51h2qMEL; z9S(X2eJ5C_>DeWob1R|5hDj+#_rtJ(S+g(mQUvYkHzXnOO*&wHDiA#`Dm|m8hS1Vl zdELfET|P#Ylk%EOU6Mu9khbxB52po-(&zz1UKVO!T~iAD^BE3A(G|Jm5<86#YGL}B z0r%5aV%7E2KQs$Dvr;dgFJBPd>hzPOHN^KI-dQ{n36VsloRJ>8YT|GtZK1o8SBam6 zBsFh>{lulNsjm%LAmN}4!4XF;G1n*cY_QmhtgPqNCPx!xb2&lR9&{bGSBZAIqJN-y zo(iE7Yk+Q4X#N3cdH`x`!z}Fqf|tZOK)!uOVe`L=oX$T0c9fO8tr=iv?ro~Xq-Vev z3B1r(`ejWb`EEh9^d;qLOr40Qn%XytyjAAI7V&at5t#eh89zM&q0$-%r$hmO9t84F z0s;aSM*yn7P)7hbL>|eol{1nwPonV}3C_=+1c$#*%u#<;2rGZy+-Zpb6@P`< zjZZ;*kA_1(5#O}bjgq{QaO6M|xJ78a^ddXj;|{0`gnv0?*!Q-$%=cqRuinJYM#=vY zNUPy@KuaZmc075qX6(-9Enb=As^ikrx+Z{IFl+y;Y`izwx>Q12vhsXfyRt_c1KxW?2O{)&1Kd{l&z{uab(c>`+dW1VD2`;djJ^sA5m=&=muJ#i~H z?CJ(7gJCJ1#?6<#KG=wJ?I*$F1fYW*#g4ub?b;9U0?66hiUN9}n6U*!w|rMwF(TUz zT#q=Zy9IIXct7%Vz(pEt$pYL3-m$4#{!*#fFP}yIQv4tL{^#O}4vBtQFlSFzq$^=j zlaTh-QM-G_fC@edOVH2}Y-8DXHRs;=3mW&LsP^My1x5W#skz3w2Avc#Q~dqs6a!Yi zMQy#F<{{gUd$(F`m+QM_9(jZ6%x52d#8bb-+S+CSanTE)`nxsbWD^rVkwK@wUY*xJ zJ7@*zHaHjR)zAMlc#h|hFqsh&?h63*M;Z{<2;7|=C-71xk!skci}>c9aFI85GBC8Z z{0ns>D#^3`@JasN?;jcOqrFen0ypQ*nq%a*5t&;ZNo=K{O`810(GQ5|+yK!X1RR_; zO#@mIPBxbTkXkLDZk?ZLZPm|!Cg61eKn#T_z_tc~<`6Jg^!0~@&-M)f{F~AM_zl(} zeXb!w+{o1w?tzNx-*yVD8P9jLPK3Yn0g?;ff_%KZ7XPu76h>GNS4=T_`>Rsgb85HU zjv&zry9Zzr|IUiw|0~lFZI_~UX3uOnmoOa|Rn{U{*==tOOzVJr3XgQ4^5hoVwaFxe zLpE<-Q6(i`UaF)UDS)ligxpv$mT56}GIdiMMyAY_QTqt z-R7>`7I3yNCmVbTGq}WG(3(#qi;?+9?%nh!Xt1HiR=}jTG6(!WXxWU|`#TjDcoyR1 zHgyp(&QlV`$#CQW%#yp{cOeiM3>00s3Nd@p4|!HrctTdh9`clqidjFC?e|gaAO!Sv zuh$!18!(#67{=OoIrB-)MC#&77gI}?)f378c7Jv(j0Fa-(-`6#?drB!cKNu9wqY%hJ$P7)sJ&Sb87o+d$Yf6~d;WM(%lC$Ya zRTDKgYdc7K<-WFZ>@DOD38{MLQ4-(u@GwvzWd;{-QAFJ>qlc zm9RjJvN-NU>7ShTr`#rc528$TH9s_yuPv6nzAO79S7f~g0qBj@YIzCgZXanOSSO`% z3D(U$-+yLaA{G6CWwwJtL%Q#mGT@-fIC1#&vue8$fTiyk;a&XQv!S+~J{GwI@s{NE z-nUgoj8p}_6TlzlhzRz9zylvyb7Oz2D0XMU1117k$rV?3QuN{=slEbOj8p4W|7cWm z4@^;Ud~i0vuVnlE&4$KQUuOivj_(QiBMD;T^~-pde?$ZU#Q0zPt{pfekjEo;`Q#k(b1bNj0^)1pM3R$GZ2)4y34^;D62=LV@{w7>5{-*YlP49Hp zUNhxlcG2ywQL`|1k?Wu2C@3SHYkvqlw*%grVM;JMX!N=+$dDbD5W~ZAO#eU}%C^`O z1nV9W`Tq~@Y3_P}MHk#c#;svx^9r%IHc}z$W!}Ft!rEmc2k{pMIjxqZ$Q4MoQnL+Z zG0;cJEu7`E%whU;dJcHi48C1MlJMJ^i=>e-=`o z)OWP~u0R3>Rc>7-zS*gf$O_?KQ=z-SEoJ2JcX=VaE0U0;tIj9tj$ss6hu!|HEy?%S z{SINJgcpE)gB@+4#r6@>#PdZaW?Yrc;B?J=^dgSpz90nN9j;m|+@oX{ z65ON^c(O(uAUxK2a{B<2{WqH+=Mt2~(2|f7BKCjc(>yD#HRX{72L;e88C<%gcPo++ zvW(~>9?0LW>&;2NmRs_g__s=TgDt>)?qMwGl}lVjTN9>S1jV`x8+KzM%s8`uBI)rE zUi_UnEnshOBSrN4OXDlUM?1BCcYyKgAL_-ME8t#Z8}A@#Tg8n~B;lg5i!=!BdaBW|0YD6};sl?4KG&F}rs_^0AvlCT$G z((Eq;^U`AHNOg>hb$^1i(Z9*(C`2-hjANujWEnm?gq%*a=Xi#XQQ-+0GCxI<=lCqp z2hhE}#=D>OVR_5< zq_%Kt%lCzQ&oNn}=#1DEJ!m ztbsz^O`zxyA$g>58Z$Tq)dv7n7tClGL}(`3e`a~$rCtznR;zaP;#0C98l1;{rqAFv zf-yJfE*kNA@p8>~cT;q%5#_YnNck4u!l5_pn>EIu_iU&oE>VLpkHmPRj(%D|lg=ph z)6z1a$f2zW?3eSmpJyf?K#&Mw^J_=qujRWzL{QX-?xl7yiw>=BIW*8k^~a7S{)`l@ z)Q$;`uOXY<<$eW{Nz7@Vy~vLZi6@hR?`z;*)_c#8it!RSTwD&sbDU}hz-v-1e{`jw zP^jH-vv?r=a%!p|K(~$f`PU!XaVg~{LkY9iD>D|uC`-#+tc-&rJo1LH-7y zx#9PU@E+_koHr->n4wx`Uy1gqeEwZ}__lLvy><-vHPEQy6587^{OS?;kuo^C*b7OV|1`;p6o$zsiHPDbZxbKL>`gxP zX(fh@jCHSfg1YHn@4HrzTWp+MBJ!R3#xFhzvKT_2i3h(cr(X##06^p$VEm5d48VEW z`nL1q!DQ%uctwls@9ei3%)kbY)ESD9z(*T`()I@D(}1I2h;G+|t!)!u|49w_;V!J} zK(O6x^B;dye$GQVutd41^dNJFVH289n*Yp8^ALVmnEX@jW;Pdg^KpOmW{K!D%(Bxt zrC1R}ntuE5)LSu?h=7RsVaXVR6JJ=e<;#XGLY|BLcFDgGCrNVE*LKp$hfOW}YdwMW zF>Ev>6k~Z7p?)6d;5@U0ZG`dI_ERYQSmq7cSx!@ftscz$<7^2ht{67?w8WB;_v$S2 zo?b<_I?HWtYuuO_v6>p=Spx7f5EL!ml<18=nWRD_XPs&!*d*ME5owe*O@^cGf@T-R z9jUH*$vr%qAZ=a>!WZXlX_l>nM3RUbvo8*BDb5wno#=%l!d~$vM>bOa+SF{b z3AGNe-J;!7U;6pYOmdAsbxMZPF&YQ&roDmm%LKs7k4y2@8#|*7sEy&Xx&$=9ke$Sr z{Yr!1a`-FYmtkAF?VwVVVue2{-5%*Gs%^K9n`I36Br;+~Q+v+Rg={4#!?>so#QFTp zLPjqe-c}c70@rnQ9q3|NYob(B)}Ml@h}@`M`RB@CKdZc;+~6N@9!Rh|??$Y)}`1u!eHNtPHGJKdGSG z;dCXTM_g5#i^gY<{WJ(qYP&GOgs6;FzOfumHp_Le@ush1F_4B~JkPf3~3G+OvdU}1WN`OONd z%W{}%Mp4mbCT8-Ep6@g^*+{BN?kosP`W1Mczy*d$GL8o;gyC@LWNbn^s^b!aQ$)vb zN#qG%ZrKy1N0Z`ffLCfA|5PS*sVZo);}ri^o{ZbBOHK>^kj9LaHDOEWtyNlh_Fet2 z`f@w{tSm2jy-(ox4Oi&eP6C%EXCul?eP?oKKO^T7Vl>Mm$`N1|Z2q*SagCm`0%+sL ziMAvux=zkxh)sHGVWU?Ru?fr=^{=QB=Py!qryuP`IJ~N~y`EEvwPTNYN;dfYhxw7|ynpg{YBcwu+P{j{z>2&#M+=JjyDv7T`Hhnw^^j8#IU5{##nq3s!qY9_U{gHF0 zXtX*jr&xxys>j{3S|HS^7*Z^@9W^g3%*n>^{D&b%PH5}p{Z#Bms}Ng#zQGW4Pr_U) z{zPE(BnkGJ)gX1zSaE2vsrux;Q>|3E8dm)x9;!A zEiQPv)IxUA>w&@V$A3%Idl#*_361f*A1v4LN&fC89iPzJsjUV>%C9&9bw2GBcOz~_ z=LsM6t-0WlI8g+t(~FP>$yAvh9KIk@{xr#6b#C9FZE0(}-4DPJ~Jv!c8M4m1TM!y>f^A(p&aqs5zTSckWH~fj=Wp4_IkT7CDSA zL#>WXt1N4qLDWwiT#u-RfJ)GXKiMg82krn@c%^rJQc*zHo72!erv*R2ktnlhxKf_= z^aH+|4@4fpI5}Q*&zSnO>W30m5fuu06o4#1_ftdqO23?_j!TUQjR7utQmR13fEmHR4eAG4;5(K6 zDabqWvv*eIjTj0@XS_`FEq?)IicMdsvOia3|Kx2M60zhi^TUDAQghpwDyNlxV}3^L@<9m_P?XZ?}xw1aCJjPYi3E zi>YwQybtonyc@9~6hBRFGz*j`<+jw(N=-H&eLUO`D>90$ekMkl$%YQnoM261ZBkB` zxX;Tb!xBc1J;{>+6lrRK`Ttc(^t4sXYK)uNS{Hw&)fu>H64KG|KwZ1v2%u<6*o0x( z81&LyCeWtmSDZ>ck--FD7S537km*IU;llCdro&zO^Nc!Kc2$?wf~=?}hZRV~2kOOxh06vn!r!h87#AQn34Kj(L& z#HSx0K?!vudSSu80qyqQ2OYp|a4U+(r>~O-53$pZVM%X5w-8ntl)^V2&x`3J4i=$ z64=;MMMi#S{(70pXA$b%7cc3F59qq zH7(Gl8{x?8C|llQz*=1i(c6?b|0MXPUB%t6wbNij2&GKy9Ieu9p;nRyrRv;AK;$)^*Q_hJWB7{mErAl=^?v!xrTsYg zL#MeL%;1e3tq+dPyY=3O!bQyM5lVUr$O1zG{SpV;PUxB@9O zjC6}-Z6$BB<^BsjJf(>PQP`>%o5kPw(l8eF zBFCU-aYf!NWeA-m=#tpnLfdM;TNz|vvoW6m2Ambx9(r=A(t`0NJi;=cg=hmnHn z*&^sR7}Wq_wr?wYN5`#mASOXWO3;-WIl8@F3j2)0>r9#z16#zz`z;1&=2uSC+7 z1y4lp*W2v7h2?5geUm~&<)oTCKR)BWh+ct~N&nY|5+PWKa4_VLves2ECm*M2SaGk} zKFEyO2;^@M2Si=Db$z?YxBO4d{N7c3Ko-%2-*nXaqtd6bZ-WKEFN( zXYKsno=Y>Hx*FaFC};#Q~v95k)~+}LN??OB7qF+5B<+^#)DI8!Za*j4K}&}EKQ z?oKBb!xC?rFvmJ=)=J)n69@dQ*Vz^fZcn>l1 z2YE2wze$t+@ib`=swq8#nt*^|LrR^p&8U#-RupD~oO&FBGT;;>Iuy);8p3eHkBsZ) z+WiXpGxd!n)t>WAc*l45MNbNo z_qu5z8>xtv4?J(p&j=rufR|t}Hc&qX9BBBGDITdzFI~A}wvxETldD}v)90=S!`+%|PU*}s)Z zo&H^RAn~%spA5`cR_A&_{0nv~!)IFT&VSsb)~`>RT-$swgymVqwM>^v*?rwLO_462ZZrlB+ExbXli9 z_(|o4R$W>nVKRt`VCFtOfT|=OQz~Vwfkmt$1H?Ydk+lmFJ(_RTnc!VEZ-8|x*olz@ zLyQ`CcR!MLVMMpnlr6?xXug|Z(%pxOmq>VyPhlO-gsPLmOZmIN5VrW!!N`nRW*dGi zko~fPDU@IB^bS{BCdrvwWDS6P*2t^IsDYSLiiri2l+~ zaOU-a3|M8-x=1>yDhQE#N0>_98=O&K*X{X&V=Z>c^@lI;W$Yj1@UtnQG^Plujq-;v zA^wJ>I4sb}4aqI@QJGHjsj*G8=f}e-YI%*JD*aElgbnsL0rZFYUZ+mt@<5a(666sF zG^#IUi2~6871p4e-|yku#_n1I0pKbFtpCKJgd=O8wcFLYi?XGi zt@#z3Gt4I5S>gG3rqS2cJlkhSc@l<1Dd+HmkA^nCvb~wiK>hiB9awO?3jVN|_Zfd1WLuy70}WNjn#7S(E%FQcs>aj>qB7 z1Xj*uJAWF6=Lfz*)(s&Dc~_c=a6mVRvngUVG#f14VS5G#rq480<>|+PvIZ!l)`BCt z7)6*6VlRlX7^MjbiVQ=>S`Wx3N!!$&lxpTmEfV7RdN7q5O`#Ud9Khx4B1{Pu0oZHi zj)5}Zl~n+LAuAuQP#a89LZkejYUnMxn7P9<`1ERZPKxc8^>;_KXl%y@1?l#L+^`wF z3%(JWg`VNoa&-&R_9=7K=t%(e=(eqT%tQ)ZB{zSD+>n>(z#4RL$N4mZhj1IA(#AKq zZpDzx#7!{SjyOiU(OlkKLP(%Uh=7SjK?@!pS(ha^<%2!B;8Ku0p>Wd#unn@mz@W#j zbP9T)74lCvyydy~fDH8%P$0XhGS1dycG0-*8t=dFrL1T06@d!=f&-T!713!g#*6ZL zDr|R7K|6#zkJ;&6*x^@fhti!w$cwT`HlG<5LZjj6);~_*SijI!{Ob`^-He%aD6n*k zyhH6R0h7_nRI_>s0j~Y2bPeELnY~^pHWG2iHMwg@Wn*kfz^eLT#;?7%(E%wDG%V1z zoXS_7t;0myV940;gd?lR`%B3p06s%596i{8Y#J|pF9VAZ7H3S5PHk?;fHq3jkj0dpzi60f3U6@h|Hvz>Dj+K$_gNJ`9N?p zK)HFsTYjR85D#68T&52BqO;CKi*ec)_?B1Y4>Sg=TY6P4!O`Bw=;>-h0SjgMZOY-e1Wjb8 z9=p%r`@lp&TFoLI_Czbgg`^X@n-I3bAMd?dY-;0@9@z05geHWfG@|nFWw$G6f6-*+ zJUi58h1Di(+#nt+b?7O>A>2uaqjY1u{-N}8*N>!Nij%|5_9DLfgskn%e7kmlfn^sO z+@Kb(iMJEZS1RRrMu_i`L$gAiEv$VnoG3B>S{t*|?DbD;o$nr$2x!@aGU(L~>+Ke! z8758m=V%+P%4P|?nK!Ae3MrWUJ1 z?pID6@wDhuPf{6ROJ9RT%{BHj)CnI9l0PF+xxk@hLyvJz9MK6=T5|DmQ=T2t_Aib%_a~hfa4tA!QQK!6@31NQUU~#Uwb2aeQ|MI@}OR8e_ft zCWlh-6h&U%zv^1+ue6n^2SB+9STsP#7wc2Rf4L8}bY!auvQ3CJ3^*^)1hvkS=sLmi zWYSrJJ!b^7I6X;N`KwrT2XsmbR5h5p5+ryR7d~UZ51_>J&$;GdE~)r zjtS;*GFMCNex+mKGn7DG5C2{8)2tF7@S{Egj_nOih{q=K@!D@Y!*Kuz`#njAa~>2t z`@!e+28T_xt#n(kbVtJ6FyFAEH>L{dPi=!ay*i0`v>GkZblH#Ah)rp`X2_Vf*8FZN z-+jZfhiBQa|6h@K=Kn~y8BnDgKjZMoG+?QC=dD6bKL=#s=T-8L1%Jq1R_FhaTc#$n z%@8W}d%g+Bm-orl&GZ-stIhIXWj0G>8J)zl%p2+sJc;rS=QxO6TYT3gzl{3kV#??4p<&Sy>mXf4)xUJyih2R)-(y<>fHSuO)ur9)GY&w%-pRFFx^|6wJ z```@kZ=N=t&+kpgV*l&uju@S><^o|K>FM$**!bz^_jP-B{x7cA$NME87kd{B0O+?* z2f9J=nqCBL4n3fXyoiXHDGEP50`v|IiCNF;Kiukx)pUflAVT!JC!}5M=D-C}JxjNh zDOS*cxyB&6&T)KbQcT0B2j?&vTHa#S#MlpgiGwd24ClmMH?0m3AM zk`~oIN0qbw@3a^{&iC@XT^^IET_KGGj}bNvH#8P0B(=0H7(aUa_fOl!kmD7ajys=l zbR+n!^aXbX>3r-3LzHIi|S>6`G{UDA3U(p!p((w1A z#K6xjYzEh8ALD9=h{kksgWNw-ez)N{S>2$4a!^ub`9dIXwrNRB_j;X^W*252O_Nkr zNJlQFI}MTE6Svs3{F1mg&t6_$tJ+jlw0X~4R%djkAyM2zMBw&vSX+s|q07tFw@JIZ zyR(pk3zgkBgm~Qj7bXNGmvm$vbX;+we|Qm&Imtz;30rKGOn8!2sB+vD*_Av;j6RdK z$vcY^8Ov&tx8osAjpuLp|KX`IDq#>9l2o?y;!;`eD3ZEn#W@yvJKuO@Jw^O}?tSjC z4Dx*JHF?WL)$FYmsfO=*pf<6)d466Y6(A?tNJ>;MqjEGhjENLIehBFr5|G!JAN~7R zN6y4sT@(6;gi;E7ths~TjYRxJ>E`W$jTqU!UflFjM^b|JelEW9xxIk}qKQD-w()bp z0(l`%!ueIm0&kKJ2i2gWrk$UMm+#l}@!;D>Yww?DZs~-~JuRQ`yMP@7fvB%-XyZd* zf@2n)O{>A0sa}m(?M`|hN50bam-+fXvIXRE_K8jp6p1DW_3#)7R-Xyth;+YS^9`}* z+7O0d*L8Mew+sF+P6>7TBS>EOB9zEoVq>q^@-{q4<_pPM>yb=Ny{#_^!Q~>lR&W88 z1;&@ZQJN24ISkwhpo!RMC)aP7b2HrZ9%kb8qR$IfrWL{Z+cDh*JnsiM$&1P zVWF%-{Lm;98^=+nuhk^Yj^wFTZPfR&jMMHY*x=M8`go;;#iXnF%)Jjy>C$%OiBPAV z6)FN(n+aSBE3<;@_lw}9!T$c52c@aUh?cdzQ<eaGjSI|JNZn#zMGYWi*)YJW-MS_W2U7tQq)15DCOD@eGaZVSb$GWwf8f$1dg6<9=F-vNqu zT1lPk)DcJ2NAN9NR_}gwm2-jZAe#PM zpW$B%{&`Eb|A+65_1eYFl^&z_Ra)(+tT_AFII{7O{00LK#P))a%I&i#XNFbbN83PjejClrD&6wEY?-DcP|A?b4MFJ%K(_2J*+t>wN}3_jYV{tKwyx zl_zLbfYonA`lV`TCyr1Ufpc5ZWhrT~_X!HOC8BZLzBC1EzFJR18*jj`m1U=5p#f=I zM@4%@)MHXcmi+&&EN!6${=!Jlc=5Z(rq~p53qB<^0PA~D>D74W4y(o$RP`l@QGF`f zd4-Vt+dsaRFkiG6z64zy%@S{4%bi1SJYHL! zr?I@7e_i#`<9(llOix$*o@J%%4|BHBLUaBOi9aRvW5^QV@JimsC)K{AlCyao%Lm4Y< zJLai>ii^tC#pkZGZ_9J~zi?(C@s(Q+`EelPIreb~N+T(ykcn?8{uA|{N3_@TRuchG z0ZYRUDV|j4i6GQ{xbJa?O@-9iq`d0 zgD^rOzNYnpTPnzccykGJ2zo2hctO}(gf5jp)>gTqRV=U%PTB3mf)?siMw^Z36+;9l zzz+veJ#bS5u%}ciWLF(|Xy{jb?OI)m7YVt}ID!DShCOW$um)$ zd<+Eklt5p*wgZ{|`PeM1%>MsoXN^i7)b^$d+=_@tmSZ#QP%EV-@Za(|1Kya(Sy3Ts zT%473sbPyV-l0@!f2?dNp`z#-%Kt4W#f#U)q`JsQHZrAzGl#8E*PCeyF(-qITvik5 zRF6EjbwTZ|YOM0g;1XC*dAy@?;Z=V5chT{@D_KI9j5G_Y%@_FW%ZQ8Vw+kH#w!PGcw2^u|IiPy_`x^)Lp77tbQ(*J- zrsO>$L;*~Z%W6Ei#2`RhV#d@fgQ=x^Id5Bt@U0@{zFj{A`79=3Jb1pHMl@mUMH%40 zQyCECywp2~{NA0^CcTuV>|R51uBxRJRBvC;pjzEcRz1uH&&(bkNzcc<5N)Nf514rhMfvG znOA)a6t;$+LQ8=)!A`^WF3S)Q=RFXR2Ovd8T`slpJ>KGOL2@8LYn&acH7?6;Cm6%H<#+%N4Lw(bpt|0fN}}>w`t%foRoMLIetuD)px?P*ijtP{8Cq zT6E{kfHb@=lu86|V#wL>Ko^QLhi<6_-x6P!$JT8U;Igoc|F|zd79}v?I`lc|tupbT zqNMgdK@(wZw|0Ywe3>8dZ@*~b)9_;* z8CVmwNrA(;o8Obd`o@#O+m7$Odi(~=V>)>|7hoRnE*?1~cDRHF*kf6}Hy6+z4=>#i zhs6+>hkcA9Y6 zdf~Wf4}agsQc!*}?hU$uiz0Y+-9^+!*vniZ*NClB3HatqlBx3$SY*hgw?s1XkBz1( zfTI0`EXU@NLeZj$g#47R)W-1yeZUD@OxgS0a0WJalUDqTP4wJDJ=^T*>AQs5<4Kb&v zd&V}i#ISE$WV>5u%puYfrMuftbRVsqzItcuAnq0RiUf!AkD4p{^wbrQ8oatjykyIY zzoIrX=(A0AXVuXE#h?EZ4m~Gk<*SUWcwh$N?NfOu(uMbk(C5NufrS8Kj+L22pLhMM^D=7W1%71ID8*&+*aO@>FC?>i>52e7#6)`%;lD z*5!2~u`SN#wLst3kHl7&6RWL~O|(U_i3@atLMD<9BS8AV@Rb2tGY}#n$FM(;9y89F zdxI$n0d!rNT%vtgzpDZ_60#CABh!O6cy)cmIK=9{5OPc>k|u13LYV=UGVIVgP&hqT({! zotY%iVW|Y_Z2R$e%hNh7ssG#6^Yu!g?Mv&lSeMr+fwnj^(gJ;7zXV!cexbICy=)(3 zC6EvrV}wwrR0y@}oh>()ceY##EvNt6)${dAp#}QAE-AE7m)9wUmeKKbNul}szJ4h* zf8QrcAr;}*ZWlc)!rztaa5(>(Q3#moR|YT8?{&%Gg}S^>8N7^+uS*8c-}m*) z;Q9MLQ3k6huy#HQEJP!6qeM|J!XB$(A1E;d(%w=+vZ)w;5F!sF3WyLw*$0*NHOc6| zGf}dQBFaEdl+=yCLHeZ5%j!!;CKl7$OTaV zV4tH67X==PMWl~z`3O1QFo8i!Im)+-9Ob#-rKRrZI$DkMbo>btaOfdzo9UP8>g(@d z4|y+T_J`Xcc7_1Du56&99;ry*EdH5-k4wL;Du+U%KC%5k~AS6N2mg@iLP;e4~gfYu%;_q z@o&m*$I&G2y$?s&k-h7S8rBAU)hJLbbM*zf(-$RPiNtsT@sZC&mykZj6tounpF`GmCc1xQ>>Wq8dwaW9b*g_zxo z6fS%tGm@J0kOTRGVMBB;;$>VEgh&=CNfWl3Lfhu^nh_Bc2Ph06Atu& z+!)mf3Ec=6lO7sE{#(dQhR}Bcr7pR9g8mC@T7@^duIkAKVDyu%GEi9F7OFJ{XTNoY3xlSVal5@Bq5(gel>~D`aoP1cU|3; z3-zo=kH1G@)JJ6p@AQ&ZCf-(Q>fJJR0f&29BcKr&je=ON=+1It z`-+xTK(X&2b%duZL~z6(9zpqzdQ$tBxobS%x3@ebZ8`nluAZ;=e7``y*LA*MsLSg- z-!G%%>pI`h-}m*O@8|FPr1O1CS$bQ_(yNu;;3|C_z5?<@fj~Ep8oS`XK(|adc6G$H zIANu*2=+lA-iSb}ep-zXpZ|0H_`gI)dV~#=hp5!yj}Y{!Jh|n1uT=R|sq%*~2#_D2 z>QpAg08*7nNSO#i7N0kH@BQx?aB;}z{XTAyzreqqKx{TdLbnkjmHse3uBm#a<<{$4#@u2@OXk4$aJ zQK-&(RQTbdclB_~7uFnuo&mxcLpRZQ{SVR;>Vqr2Y|^8f1ti8zrLdtgd$okw>zo5S zAC6F(f%p|yu)(3qAfQKRin< ziaC|^k#(B4mj#lontz0A^;;s=um{AjPJQ71V!uXdNSuq2j6POGFs{> z;ski{MwDz2edsU}BDW6CJkJzz$wTp;#qVCDdvl4VGx{c5p@&C&&G0|DFhi74p>*P# ztkS`2ZHiDH#)D`o1$av+z*U-G&}t<1@uJhNh?sBGJBk90dZ$&ttII?#EFnzq{7(sJFy)+Ui``+F4Xm<>qo)CWdp#~r}y8? z9&dVICmnq^sHf$J-Hqzx_~Y#+cD4LKyE(m`<_Nnvy`6MuZ7OqHQ<&Q(9!vy!4xod6 zQSqZ%w=4u;eMfnRaez>CHYs?$ART>ES&y2>rhIE2PbJ?DPGb4ic$V_5v32roIX&L= zzCN~mTcD?pC*Kz8WpwrNIH@&ZqEK3&Y>7&V##X9+@vScY;eKc9JKyTL|OBU$uL|LLdWtxX6S#p@j zlF~%WB6KF54@lPsQGp{h5Bmtfa3E3%YPF!$E2_4tfFdLkgZ*ZEKi{?5O|4aH$SjoET+iAaMwfMPAszMz3|(auRBkfJO?%ii54ik8F!V4A zQ%5@Gq|@xANY|N83?If~_~==R;nimi+RPqrdS4$~46h^?&_;FgamDazbLMPLZzl>i z<+;*4NbSxC)9uckcy~U0mb>%nGwN(+k2k%qkG(suBu~ypb@Fj{=hfzu*__@^+?|yZ zQ?s4go!isx&Ij@C+PG-?7=d5Ncwdh;ZE&6u6=ntOT zqF1q-H9q#Gts_;dOF8lilkW%*LJ^F|NE|dneD5BJauPRfp}_DU%jf}=_ffN%n%3rY z)7pxswauqBE(UUo*-%eA^Z|5T%rNnx2d>VHg(e-b=#`i#p^Ay)p`Pm!X)k+o6f4D; zS2MERh~Q)%JrBvPqQwhj0I`o;xejg` z^|mZBAFn#)63|Tf%QdI-mun|BpjP3cYvdQ*xxjOFoh9q6wFG3G77%J$PRuDi(Js5g zxH?^G0iruhW%yw-TbClNS#1q(?idPv=v}4bDaszdL{5mRX_#S!_qjO&;JA32Cp7~b>7N$AQFgvJIl zs)HYKNLkfvTlnh=H~+Xpq|XN^yrvRvO4U;Obn)iho3qO|-4AE)-+Y;&f)c3J^jD7R zoq8jdzp3#aum0$D0Iy71-TlUp4KMCI1{-!FmqN^1XC8mhyRuOzNjLOyw~V9{I?3NaQmVBDV^lnx!G8jD+`B+UO?y?RzM$ zdha5J!V&h7O7o%@FK_*pgx9<+5t1gG3yJ6TpmTktw+Quo>Y?1q(0TGi{u3YZbqZdY zd+5&2B|_lSE0r=&r^At{_J3QF z?>|NAzpr zhh!(^V=6&}Sc#%HRF4qd1%xl8-hDx9-_!WpG!+(ky0h~YoWl@~_~HX!!D~dF5X+=Q z{4be9;p@&3Wq)y#6z?$cU=CJS90&w*(mrU;FZjRBOVp@! z6heo|rEW;MR~NLZ7RGxR(5gTf5(iS=S(zkiOt;`*5TXHO=p#{6E3Wf?p6Od-x{7@x zPto-;P2Wv-z~C$B!hj#e<6zRS;P)XC#gdGZ!>9ZklS(8etjxJn#Srha5FuF`QC-M4 zyeCMj(4NXkG`ar<@Qp%G5Dol#@eTw8`%*ee&N61qSXSzbq8_FCq$t4|6I4c2Vk&~K zdXS1bRucA78jU1R3MS$QufT)8lo}$pl9@-;#GERyO&%x>Zn+g<%~$o`WQv>|gS;$T z_bSzm`C|M|p8w(qNO7ji-ZeJgdfE5t^v0QLN;rh(pW`G|3ZXHxA?Rq_I+G0wnJjX~ z#!+#M@G+b;P8t*AKk>jPA^M4VWO0|+o zc(_E#8ea2Lze>dRUPV(I+o=*VFz$rjRpmJ3;9_V|me@^ujbR^L{NqiMtIeEm*_lh7 zpl!iUB#GXp|F)peZ9v23O0^JZrF`WxoB^M4gF$DQG0 zt<{gURr>wwHdSQ+=bzs;tz{|~?m|Z#$LP3(joH+L)g*&sYPIa~L0q?K`52-eI&(*l z%dhIAlRC9hzMZlUE8*WMzr9>e8v9XsuzMeQ3?1P;x$G z)W~v~O7?=%X)9GukyLU&Id$Sh4t^)$b?q!9k*~)051-un;4P&QpF?qGR~bx+4}uWi z@OAW)v9<2({LW9XKM^wU47?yu9Fms>hL@^$-GY9*s<2neM3Xe~+xaUHAesHLhuAGb z>R@3wM60I$&;S1aIJHH+S~Zh#noQL>Ihmql!5_$Bb?`Q&KIDLnA4WM6Qg3j4CzF%% zwZ_*~(eXy=Ec-Y_x8ejZwCV}I zGhNY{y`7yiPZ+3VLg8whr&hD9uziX2lf}6p*C-zN$Ti>O zpFlo|21BVap@>C+)EQ!GaC|%#QSMss8o%kGJ3vBb$hR9gXrWw`8JLCMTDHniAdjw6 zd#B|ScF`yxjDJ96&x!YfDD<^@Tu(9aWwq*lUn~_8u{hJUI64b0GVkOyt?%qK>)?zp z39SGJa@z>W4VD?bVVVlQS`|X}C&zR-5U4Z3@*zVerb8^(xIJm+6i7xUL=RKbqYOeQ z^-XQo5Y}sDg%Z6)cbL)oPD|1*;c`GszQU;uUo9Lc+$to7i)*S^)q)cY=Mj-Jj}Ip4 zOMRzZ2fw8k_xrJnJ3X6ebze2+WgLP%l%5piYiBSw_MMhPD=<5V%LTo$_7-o1gx zq&=G@R~R0lD&R~H_;MHwQ^INprRnV+F=;sJM_kY=7OHXN7Q1UmPYbz`g~(yx)tNF+ zQN-#y2VyM|%F>I8&z|IM{7s>Fx-MZscJP+4}yP1b(-mpS5iyT0_>+ot?L) z;lkYP_;UXA2V%l>w9gl-KAjb6u?nBuhz`n=xm{FyZtb@=+g)f)_)0qqLraOZc(Glmo2?f|9hZMJQs!OLrLt^2B3c*XJzGpRG%yR zIHU}G;-H!h(MaPd65;{&g&#>kC_mWWPd&6KtB8g#f8R z->s=5Xe|kDt&iM2ouAJav{EZPA&y)-KSrUT>)YfUAycfX(?!ii0UPoYfe7b~9k()` zT*{u#&sEqT(5g;YOA*@Hsew-gSb3opk!~|-%CM(kV)zfZk<3m?ZcuoG(e1uyL9P74 zo+z4*w+5<*zS9$sJiI5Uo9>QuGm~1IB9Ie{XYIr#oM8loYtn|NjV||m3RG|ittucU zcGbjwa|KA4&x9*HIen4iGw?Xz)3qN0Wnk~aP>YX(8Lo@9QGDp++nr%P+RvDd%rzlR zjOjJyXT`$U6>yb#oiFZFvD7l3U1XUVZ28*!+CIm-wLyekME3_$sMRQgAs3z__d69D zU(1B=$W@zAXq{a%V#-9fv<6BmxmMpfb(V>9>54sTXXl4lxK&f;R<-i!L<^--9`O|( zA8V?rc2kI?W1M!ej-j>ok7973IHFK{85i-o(@6pmCHjFBiI+i6k#!wE$ zvD#L7qq>m6*wh}nwE%|fUU)YP2RgxwXFfoX-h*2xb_AEmeSn{d^^uW-mynEbyhX&8 z{qlY#`%D$+14{;7m;>bpgmJAE&xr0|TufBf=;~util@HwQ5^9pNz_9^Pg`y?58_T} zi}(lZ{Ynx28F{1Ml&h6c+LpJG&J#GMovfPoy@(^>0}+!(gcSzVJ-uV{_`513~ zzLs`&E=5I_t3qG0D?StQNS8g7i+!{FdvT6usA64&);cR6&UP_%6u>-9z9S1xZT$Tj;a6D4Kjja+b8~OOE5`Lw!yZy8-ASep7-u-K9O_ z970V0D(L~1woDZG>}h|+qA8fgiP$&oHnm!aB8qCtgkP&st`vMu`T@Rp@qsY( z;>D>_beZnQ%8VjT3leS$Qe4$)Zg^kgw`w&P^`e1VY!^uR>WIf;TJOfRQ*NaQym(R2 zp%*Vq?~0=J#G8EaqL3mlUg!mtpISMOQu`$=UV6cu?(F=xI9JR04njnu5mGvMRb`o) zz9Hbs^!!N_R66Ojkw;t%LaW9E%oWQK`|8YKY&vlr1qprx=a}D7&_3=*{HoNjo%3VA z^w2;d#grl6LBA0%8pS%C{fH*VyHf5KyT=7$5c^la@gN>44a4cq4gdfcYnlJ4V_SsU z0R%Vu|9AX-rasSm`rn`yfbSnFG(b+}^q~@D9=br{Q?iI#<+LI@Q2F-ld;L-Qb-7Yc zU%UVx2|#zqiIm4T?2((eLUdKvXbh$KLq%L@JL0XFXB4UZuaQ!zDJ(!Sy} zlP3Hd6yp9pIR6dwIP)TKA=`56h0`Xtp}12;t0A?!q~=<6Ex2)v+19H~^hwWou8DXe z&tfB#HnQj#t2m}Au5BsG4MOZ^tD42ZO}t|sVpsa$Q9R~s(tP=BVYHNLkPkq50thF) z^a!*7aE63Xvlu5QsIZX5^CrT}c5@KxJVClufS6O?)7k%}tRnTDopv2u$Yz}=6uxCw zBEUef%LVGkWj&a#+afoC-@d1!EI_2JE_yle=gw{*_P9(QPp8Q&U1RqF0)J5AgIe(N z`%e!Q5>!q>MG~S?1(g7@A-}TEJH4+vUHSIq`?AIH2prDH3Yh#%RM(Y->V=f4N=2_E zImgS04#x2ioY?296lT=)?m3AJh5FjRqe*a2`vB3T%hG7`c;Y)M0|DjCX#Lf;CpuWiU9;v%Xl z=BPtPHK{7(#x1Fj=t7WK#YL3tpAu<_K*oYGpY8zB-lI<(tNa+VSLZ@I5w8S0R z#rM32r{GiN0zvop5Hs|P@1V*T@ks2YFWZW}Xw{!_>!+a8_z5Sm7J7@*L5ndti|0s; zmA4SnL!1y&J-RYUTHo2p@2g6s?+lT1omxs~|KgJ-b}A1IpmU#EgWpqd{+pC7X}1o! zkmF-%;+r6jt7rexr&JEiiH~A?n{c9mLUjkA;VXOgFAlYo7xQmbGss8UR{9W&qR)Im zrsW~=Q56h1XG)ACS0)FD=fBmpCOLh200;}@#X-KD1J$T9JCp0(!9l2iqgo}T*8H0y@wdggnIo+e00js(X4ihD62JLTqY)fRG*t^ zTr34rj>{>Rg!))Qi` zGR%0~QIq#0_5{n7UUSH(C&JE!H7mA@D4;5xWW=y&k=n9qT&$)%x~g&5Bc$LKd7fUh zqAM(tm5E~gPuidh!f@4jKeLLhp{hH91BGC=e3Ssc$VKW zeu&Urc~zJP$Ctm!GM+pd{wAZlvK%S z1~Ka6ru_GUBL6)ROs}U)N#&ZvHrVq_l~hvxSx_PsBPGge6QOy&#dMfNf*1#Ixv`N* z9_|@-t`VcD5Lph2KJf`!NOOpVSKNN*q$TYaU>KpmA6dEWlv(P3v#d$-`hu;!r~ zYn=VN{)XI?z>9Y@E=1!icSnHxj|Vu~l)C)>$9Z8@o1+Up=Vpom>#9evHMV|7R(hQKaw^P%mJn^mAt{0!Kg<4LLe*Y7?NbpPulk5DH2iNGJF(F(9gaiA%H*3}`0kBkOZo2R{~t zOr+P`r?kRf`?HCTPi?_T6w_%7&VC%ev8NWtKh)twohQ;pWqoo{ryIFe5;Y9=&ABrm zCIOCzM6iFnPh^|YSkX8Und%S2?@Rch^KB#cdx;&2*{wXEiIF7d3Lp;_G%g;0E5wn9 zuSLVx2+4klnbli_!ZC?10Z14noq>y88I)v;A%uEECr&v3?Uk}I|3(wdy!IeTtXU56 z`ACInF}rWdHIu;Fj2kEPldqrs$%*e`dM(|H<+)tTWCwa4N~wfvu_Td|+>?-`Byd#6 z?#MnH_yY!5W$`!#H_e@E?7OEqmBP*lG03UR5sEK>ad6|*in?H4Z0u-gZNldr0BRrw zSNQ{c%&#%MmT1|50s`|FPQeG_%QGk^$%h%fuHX6f;_WHer%^ACLZbWqPP5fDmrZoFV7sN9-Y`0}%jprX?j#HK}zap|=bGx%rD#s~muyQ}v5-ibnioIAl z0gPh(wu+Wdc3@!YwT&HEMPqm*azV_m9GqYehJ4+H1PT{GBRHjQ$XF4=qfRJFdWj<_ zbls4jxOD~->EbDD1k!%V2)JstjzmWK=4rFhIykwK!lV!?zjz^Yi@bP|xY*Z)dQpim z>9(D8LiqxTlc>7*1NPlB*{>_*FB$uyT{T2c8oDQG9V2jv$Iqm~fvCQ-^ICvC74nht zB-ID=8~$P-s$EMQRp;S9&`#MsW(h*3`C?Jj$CBt<}|&*T-U`x@kf$7{aEpZO7{-o!Eju@Exev>*}}s ztet|)iV;s1J(rrzRLg=Y1!Ix-We4M#8Mh&yniSN#Y3`ER{Y(Wpr}keq=qDeHzYl|; z7DMV~e5285G!73A#J`P3BlB;w+3K|Z(mXgk>NF0HI!DdFG@6}e>*z0_v5b#T`iUrm z;a?gHe=9q)aOuD8Pa) zq6ywI9SjlYtI=vd8B385L8Mh6lsQJ|Y`(NhRV5}Akq<&6DUX{zQNHRbT~eXeA??s; zxL{Wk!xo7`p^q@UzlO(paGUIOx}dPOEEQ908w7!I_;QUr;ZD=Q{#ol`cmQOxbqy$5RGOyj=T zYg+MI&52~v;UZS47P#bBGd}s!C11c@UH&eYYJLywtCf*T2DnVw zlxkRfZ>D56X4@{hsok{dNBLAX21LuYu3FfWy8?}kCVEG{fgvUlE#@2mosdv!7##Ck zH(>Vxl2iAnQcCV}>d*YbROdeBtsAGv)T)Ez3THs1^6JJ5RIOXhdQ)wFagGF~3O7h~ zcK#NqtR=G8vD81 zRpF@)E|C+3nB4=R*Am8$F)kbq@f{Q}x=+<>`vGxl`j49QaNFNAOpm%STpUf9rbT^c z=Zp$<2Ne$CFJ8ncpvMgD#S5@YLTMb6c69B5C5+-ZO6Uc;S5-Jf8U_Qo@h}1c6!HlG z&nam?HKZ}8SH$WB`oK}@H*TXmPx zd6^|^s9i#^rVPX+p;#)}eJsxWu*gOh$Eoh`Ws;D_K5(Uz5hsU}X=hWj6MFz9(;m-H ztP2vyB(a`S&f3MJlz>=d#7{kN$)N9wK*5Zp=*!|3LizRgC}>mij4iaWzYx)CIqL3HrCVFL{#4~c9as9spm;40x!2(FW^XgjS|>QrNgeZFL&$buvon*bK# z!9aR*CeqYbxi%Ocp*r}~ZXGla$Kqc7uW}p_FMKEdQ0bg&$F1Yzw=QJAaDv7)+7_Y0Cv701+ z>`K7V^&n;45P1Z;T^b4FXg~70BEFTuQP8ExA-+o=^uhJ7FF#izU+nsyD^;^W4^sN_ zKc6dq%U=9ksm78_JP`l&x$^B(yFtI~iOdbV*%wpvR#PTX=eRi&b9Hhk|4YxAhewz% zvu2}FFq6`92QCCPQgcX$mBg;Xn_tf^|v_ zPODD)pxrq+>>M1oTjH*V{p+Ofl>qw4qcQX9)m(9)GejTp2$6^xUzGRwjQs$e>w!Q8 z*?E=sMYL`4Eo$r)K+pR@rwTd;^1nv?BsKH$iS|(yG+QTCaM0k792Y#(X;eY;s8t2L zwMPBqXiw8Oo~bFgw9=1fkp#)SR|YT|pm^64mqzeT4>Y|jN7&al8kx0Nzz1v?B040V zD=VFhd*VF|uaO`jUl-Zks4@t#droM&m&3{_Xd8|3E*u(v-(`Av&;QOfjrmNs8p+>2 zL*WhdvdbgA!W5Z;0Q-ZFa^W|NuBR78(s_f$lk`F8ju~?g)Ggk0eDPa#d}E;u1{>E{ zD`_mZibnA6HDvIdU|)z%WA-bY8sf%#B6dB#mH0v@RY^Vbk$*Wuyns;F$9zI6mF&eI ziO(+apQ6c5<9`|VMQnHG9dWJn6*2txFc?F2erDqD@&2U=#AL@SANJ2OH%R=#7~0VQ zhSwhUGrZKto|hH|?Pe7;o5xkqJU-$4+&oUPe4l?OCrEk2W;3#t7T+08+Iv~SP{v1M z^L<6UXym8qsti))WYdCN?2WrzoZ}EOR4&Z5bK8S)Eg)`28ZSp)w?<#~_BbgUjg)Xr zZF(F(bSVlkqUlF8$NvcV2|Eqy_?kj}A)YFkRiR|sTB24^r`N1ceDq8t_2G1q`nY8!wUyLXQcsrDTt>s2L6_X1P#m7iWwpGaQj~Vp zYQe-`p71t5Rwl=*Leu>6a|JpK-z0*44uMV?%!^B~Il}%`LSifFt)xE%l>mhf@>%3# zwy?lIUX{exkw{=UBEQ+1@^s#`GT+L4EAyw5`I~YqugXkBfBnzr%BO=`=S%%XEYZO} z*rcA$k24+zOPqA7;N*n=SMQ9+6_?Un0xlp!VAm&p?ci(Y?hb^A2E?bxFke`>@%-S1 ziyS!=rI|3z?;D&lYlbjKdMsU+oz^L9529O;|VDe zikt$TwnS%yLC?x`E7L)lOm8)(l<93N)2&RmGJUd47b(5w^w*n3%9l%T@bV>SEMfsj zQ+B;_`yHgZv?cfvV$WozCoQ{6Pvg+TkiOD#_90ojJs9R>h;UbZhx~!aF5PUTuA`#I zXAz!>UAe$XQC<&2v0&0);VhpL2wZ-p2U9B1jxrK0r4TQ2Uu)->e2DO1DCVlwXaska zjHo+I{-z($ZFWc=8Xy6RjE+GM5BxLwvDl%PPi5|+O3GIt^oQ6*Z$<&TFS++G6q0;< zBY0Qv{0RF6S3^Gj%R=92V{K;Kk8V}ZTH~w*4X&7wJ)QJ>#2lfZTUd4reya&G%@V$|$Zn zBCf076FqwG9_*@%^ZUOQd$CV>{7s2EFol4XTTbFp{_p_Q2*9^*>6T>7nUFTJ$a`fL zQQd4zE31tK$cgl$1dXXR&97e$aX+`%W8wZ*4WiVKFZO3Q4`1{G>2jzAdk7uE5V-|g znB1sZ+3lvQk%AMABAJ;t7uhit(hwf_N#F0_9nOfaUL;a3XUM^aq_!kZF>?o1hnCFK z#anc|cYgYk>;65w8+Xu3)_aKl9b{Iz$at3P1iwKBKR}9di&YAu+~ZObE{pY>7{AQFXE8<{G1D9P_RtJ^>^qd?)8q@=mgA{lKP+qFlCs?lyjUq-yNSc`i)kl zntg5rDU&kkyI#N9Yc%qYB;ljxW4RTzAy_lr;8?KPSP9t7xntzfrUaEo8w4r~g6I^c z%)!U?f=WI{3n;;OK)G??aFg5=ilN{*HwzOUZkW%-^tg5vI0z0!TZPSIP2WZgW1E`-Gruk#i{6SU^jMeBPchq}@1Zxnecu zVFpI5AxwN{Iuy@wa zy#RZQZyi0hQr9BvNvj{1ZZHg0r>&r#*XR#WJadf=FlY?TV>fhmvd_0QTr0%L$@YA*$j@713_-3J)>E z4$Q>Ns7*2#Qla8ta@S>O!GMj66IdM7y*mGO9z>UO>#?qu+3rsb_EJHHz{F4$hlDvC z_#EM0x}T4S_8bxal@S(GWu{6(ikb~Grl=GL9cOS*>mUP!a(3XuJPyLHh~(8$F2^i`ar>iUr(R_6#+aJWRt;s!pUKO^W05uLsx_X>k74)L-@0up^vZt#@dd*c{R zAa}ztC%ul%4Lr9B3u2ii0Vc&W8&8dmX-=Lik81|zk_T!b8p-AV`Okm;v-bXd?e%N$ z^UtTF(P?&wMaot+<&2?F%pWeIO%NiA!kgqkp8al z*on8DiThMM!77Nr&T!ETi>P$)T{dpF7~_b0<=xYh=SQO+3X3O?1ayX}dm~C3*@-Kk zxF5z%+^VPNywd>hAVa=$|D5Njkp=L%BVx(qy;|<%xy)6ie`lJ+6FZ5yNzBjH>&=*} z6FXOv=jzQS%+=cu&v}X_$*mtB3vm1}lSH#qRxo8B(<+fEIckVq7x~M9@W1@}(awy% zQIb3Ecm}aM)%s8i4cMmaTWDvqp_umop8TQa_qt2;0bzD_fZ_2R}T>()&GeH!eMXnWWq*@rabi5|awjkkEmy$1KQ4JXvmqEy zY(5`%WBqA9yKN?T_*{BMU=M){H9vr(b1Evae+;2d`zRE##o1K0oECXXOl4J-uk3*F z7O~BS=Z20lxyYjPP#X;F*rBCj3SgH*xfaiEXkfI9*gAfg1wlrRWSl?0K=3VAXk>BsgcPi8wq7nQJ zg;W)cKa`ib)R6nN3OS8y-itPsb6(ssL!-ch4EuvvSXDesnXx8K5f=N<7Q%`1ByO!3 zQ5X*tZ9f{#o8moovw95~1ea-WVPpy~fF8m>Pm-CBCcbJG|NnP{Lg57-6!qO8d^2@* zfbKFoqbVqLN3t?K<@)65)bo}43$vXGG(O8p&X~A=>vmx(W3h=eCgQYW>SN#WA{U(% zrtsD?^Fi+#W@DqsW4OS#OV5{I^pY`Mg_9&c*_94`6O(oiAstph(Me`+&M7VkUS@D0 z=8^u7?qel=H#Zo5t@vTJfaGY68xchxWkZEX@rkY~vz917mCmxbvM1V=uCa%5Z|v0a z6Fo_}X4|#5@oR6sRkx<~Hm!Omy)Fad+EqEOgPXlp^PaL37i!*tbV;i;POiA{5}l;U z{FdnZ7hJoo%~-qQ-!D6A;mCl87?HB6f9`t>}pys&PXplMeXEag02z%_UpVQT( zMO&=~ehOh9`ViD=X+eqn!dTrz@a6424)N2KrW#eyYBsB&)#+40whV+8ZfD8xz<+@= z4~5wvgXE-6hdlyAw?IcH{#7zu=;=co==d4E{}!g7tPFg_gnsGduz8@bg)kU?j6?Pm z38gI}(VY2^B} z$R;) zqB-=dKa@wVNc3~ z*>mCSt?ZqmsVfa>pAghOCak?Iuzh@Jdx=#cy_=8s7y9f2(5QE+{9hW4lz+{hxDJ5u zPmsR?Ft;y}ZUDqK`j`I!rlCPUAd1_{Klb=Pd!PNBIjGd1qrjG6l$VdEF7IWPS}e2E z?qmw$Z6WDS%4zq*@@RMRL>J!4Nh$TtQq+roPn|Q%Y>{PdD(cP!8Wf%D%W@BxM;$4r zwZPS92#^iMFUmwvo^7B|XOzo0fPEL=U^jv#3YFr_0)@E+#$*>L&o5AxVPK44%x4)$ zcR8PFAjca<41aDll|=BD$MBa%@t4N&mqhZ9=jU;G2iD&2LQcx(At=v8FeVp4(tNSn zaZ|zf5J~7w1LQM6d=QG<(}J^yC23&QLJ3$xXA3$wQvW-E8# zM}j-0NjfKhq45qsTME-3m-J5#r*AI#whgHNswH$Rye&qz7`?F17Nc(>MqjtK`gndA z-JT4i$E9k0B$!=Z?9^iLXM@2Luo6HvT#{PRA(6irSeXT@%#5QS9snO~)^&dKF^#S< ze6<4ssR7?O1=vD3tz~iA%5a*+ zXY0ggWjZY*NoN)fStvA(SCi0CImD;Y2)PywSyh$AK$+$@8w0IduM{VTod>T>5sho{ zip4AH_6GPjJ*rz%HF+Y|Fc}J_YN?$L^zH?&s;c)M&kqVbJm|h|9VE(VhpcZ@JTB`S zWl>SKd5enVr4lvH0gf-KKpyX&>roB zR6l0zhJa)9cO>pd9%Yr)&w1Q%&nc8tGlzZhKLpWdsi-rp1DYbVwx6m9$WroVcbP&G zcxdYY>Ddhm;lRimz=>!9>Ei%W7dgFqhAbwvm^5`~TQF%}Q`c_I4@nOvLsFa1Lq1?p z<#JKwWHhHQY2$v^ljY(~BjFtIZf_1?Ik78?Va-iNm{1~>DHCIC3cmW)%)Nvi1k_|% zXl1RRxjsKzOtoWt!Yq}Qq_EFb%WAdsja|^cGH(Ond;-VF0Q#PjjhdH5_N;c#Rfs~g58Cwjbv>eF%1wl#&KI3Ab*Esk!^e*px zrx<_dYI;nn>QP?urKGaQnA#qt)ji4*%crV*Wb1sGRWDMdKGN@J%6&{I_>nI8ktzC- zDf^Kw{E;roktxfODgTkK(~&L#ViW<%R$DG81TyKX>CYUdKa>8dSrjB)79?F5#4HU` zG+{*rLW)X+=oA(|>&&_5N%al=k}4N+@Y9dI>tE=^0L*3YSyofRs z>c4C1@0w3sL?@R15I^JVe~7{#nff0G2h9Up|Kl-!II9=d@fcIPv#^54l=VBxs&^FY zw`Z!{F>T$B@~R!#(n$qnIg+9r8oXvpatO&QRY@(Oy~-5Z$d=lOH`|Q%F6r;XgFN0k z=-W5m@9)}2Cr554R}{2R$ycW0D^c)GRfioT&wVAz$fY~#qCSi~mc!&uM41pr58nz; zR3xRE$qh>>6C0@uMXJQ|I`Buv*KRX);fu<^H*%dAr#e)#9sgONH7wQ{mTL@)^@YXS zLVak;>7j~Mg~f`(WTZt(!eSNS1O?$3C0`}*q5<}kV^rm+-ZWCRmZ<y;tu>oa4ch0IUq@GHX}{xe;#yHL4RrrIhhMmb5XmGDooN~=JjRjSS^Q)U&b zvWgX11!}CMi+L4RT7RDuxlf;^CdW9I>WRvd=qpqshN5byEBNU;oCwj{{Z&~oBjFTrL zof6SgIMU|4BW*5uq|IqZ`op{P@HXilM;>jYxRkeUCFq?8ZPrc(+I;L;@@xTy%pS^w)%9Xg!OiLD6(ZIGN6XV-A55LIhoj5~PDsfj<#P7~-u$m@!2I zOS0`;An2~dT9+ZoZHO5|{z6pYZJ@^?%kTVCv)lI84!Yj%|6;+`ion(nnN(xe0yd_f z7G^yMn3c1D&4aU!EY7kxYgU})BX&zdj+7jPZ%`M7p#lYlg2y9YZx9>SUwn0mbB)6K zi?4kT`abgNFTOg^!~CCKSbs5<&o>7raVIoD@e%6=P{=gh7q=@?f$Hw~*!k;!zOTQ) z6fzjpUwo~<`0f0I`1hQm$R*-e7V&Oje^7t%{R;ihFJu?N8_~lR1HbGXX*zAnG>Lv} z@+=7h>`FY$@{`0u=+hBq8^K8=Vm%W1o6Jcc6}@I+rB};hB{w9&x+f{-BNtN#hOXwQ z&6+38ORq!1Wzcl<^m=LBv9r-E6p@VZ|h@&4j zL@Dw?&NcWL!hRn+RyVRLLzLBxSlx)#jp)CN{BoKHbLd8nt!~8XM&{Lx!~n-InJf?9 z*SGpvr!z*(Gfu`uD{nB1G$q+6*>hUq+pdc#O3 zF-%^|BrqJCzR*lwm`YukN?cfJ8mSY_G>WO5ie|z>GmoH|tT0Yhm>#K40jO4AT%S#5 z+XMUHmCUyXc0UjoQ7K(?vcJNdd^PC}X(aw@7bX5vKlA4Qn=bR8{F#vduXAwF%H;oR zw;OGn|L-w=Z2rGZ&Hrb$8OzsZ*tC9Dnei0T`b|(5hy^m`(RSm)aBK#@REKR8&a*hG zR-Y3VFkW0?Vm_OR&tevfS=7&}Gx5!crK@|=P{wAaOLy1;ndb;(EMBpBBb&(92b+$PP>Cdr`{ zWI$QL)Ql59Hiz5u(qPyeZdQZwq%;^d1Dn-g*bHn-!yhXJ8#c+>QfWS`dSMgrowZI` z>(o=nb~cZi&7<~2@~FxCCHd24%AEEzekSIB>w5^X{a)m|9-1ZToAEO~|6B8@ahT2j z)@&Re+Wc>i@gq3A;v(vVSnOg#?uyx#ESF7hCU>Z860?+WOds4c89oX&uxk1V#hkSo zc#rRp3u-kZlZTncqxAF-#%KWa%@}+%s=2$u|2N|foO~g971PFf#R$ zd3q8%QCIFn!YA$)Z;y*rDs{Hqt|4|o>*o9PME}}R@?);6G zu8uD$s>f1^e+Nh|qv?rJ#H0+e9O!?7EM{QM9PJk4tY#5wWv@+St=2PSt;M+^|FP^z z|F!%E5S%bJQPcL&ZI$$TN_6rGTPdwR5cZ_j6^U4v^t<%lcUDBPUm^xbA6(x1&J&>- z3d1f80uLj1MKpU8hTu$o;fc^}L|oio9l_qK=AQ_)0_3~cAFPO2=d$%DLaSSd*^1fm zcZgXDKTbm97KbkjhaVT<@YcbUN8ICaIJ|9fxW(a9(#_)V>F8u}xW(bCrkTazv(wDt zaErrNL#yq@;S7(E^qxzZyJbT;w}keIyk6jTkas-DC!rrdeN)Zdq>0OBqu2fgG5ND6 zh1i2Ycq(+I-_01jSOxj{&CZ|UchIU9A6?{Ui;?;IsfK;Wi(GUjd~(yCNja)6O3)&H z`A?c*zEuJLYZ+u#zY3?SZ)!1{{v2f<9a*0a^JVQK5iu2wkg(p5eAr+gqzu1mG6YF# zk|dQU?+@|ZerEFja8clq`w{ZlOr9VU{69Jetz7-T=3&SBe>}#|82^vg@oZa95bFuD zsh~F5le&~ah8~nNU%8mBisxQ_mdczGpOD#`UC0Y0;Wev=UJZX)e-QKMks0iHX;v4Y z*ayO9A=(pDu`WQk-yvp7?Xb9};`x#9O&HEby(g!3U4VGMMC_OT<-PArujZG}KYPk4 z_3Cp}#x6iLt-)VR;XetUT3`y64ot0s#Aa}i1*R65T41^juk0@F26&H~eI2d36z zV4{ILJJVH=o9eB%W#^1o1y=2Z4$sfNJ*E9uEsXgR0F1Z)wma=+Cma9YIy$oU-^chV zwEvp9GOXFxnteA4rDz!?yt29j%(@{82wgZAfNnSC^|e_0a~Xtx!u;qQ5UX!n4aZqS zu=z6+%Po^NCLa%;RBX}`cda>fmXmfLyYE;%|AsP+-FKetzEhyadBVHT1|2w>?Qz@C z!E@Y(LL|FIyAFx`1%hV_&mGy;)u#8jIms?`CoBjkq241MKE|_O$FkoN806VPfhBEY z&&7G7jBq|-Hh;nb^c!JHR~NedUbg;ZiD$C}EJD7>zxK(kzar}WDqDZ5#3Nxag#OCb z-Y=^8r%JwCh}p(wVX!BF4N(KOn&VCYhxP$L9Zjl%H|&-#_u$wn5;VqXM=;;8}brHl*d&gfDE` zR*2tjg@O$PUme@oI+e?y;$!<@P1=F%-m?+POdAYt_nwtgahvuY>wl6Xq74Xt6eij& z<0Ac+@a>xG{81}CfB3{#3crQFGVYkOaO2kJWs46uHrRY+Y%y6tek2TfPYYHoanqX0 zXzCYYvq^i9-F-IWh+_lY?CxWCA7BqT8*s?6PWB~Cr6X8YlKQIf6YDX#*zoraIg?mF z{KsOGXRQHU*1q=K9AgWqXT#x_Ktvl8zc2-DVEn=~v;p3W)6ixZSd@ZW94Bv&3O3+t z!??*F6_!a&dsJA2diJPbj|!VWL3>oNM}-Zbp*<=*ZclCY(_j51sT@_nVwRp@wD;3#AhM#JQUW#$mc|zAqsdx{BPs1)y%~IwhwIl@8kRw z$N$E&4KCztKl>=)4N!AE_vqhb59+Ets0$s2Z&3G^gx4sf4}Xvz2YP>y-sg%GNk!oA z+Z0&u;UW3jw{PI#0sE9e-$CEL%?SNIM57)G0qKL^WiJ8t!LJlO1En$NnN4P>v9Wc} zB(67v4DA+hR)`$pJJ>^A6ow?ET}Bx6rsc3toza)QJ% zaHG*>YpfCV8H9Io5c%&2MX2kdfDM-lCKqNVjvfDpcMiPfowML^?flYijpLVR5HmT% zl#xLQN8Nu%&}Y~~yN!CMQ3bImOE^mLRTK&-(4|}`xU}1goNL4ul4tyZD)7+WG*a!n z(*O?-{FRv*@KSae^hpTbGBl##lEHg^mXmC`+e3X4BG5zFAAk`3J3^EpcMqI*8qY?+ zZ92~$j8&mLgGj?do;}%Yo;_Iu&z@MG-58!7tEG7c5qZTtd$QR)d$IOF5y5NUS#dc?|3oallwpv&gdwdo_}k2)$=MW!U09Jb1;d_LhBZ2|F${Y$R*TboPEI)qUxtYBRk#FCoHc23F601a z(&p^gXrw9pvWx}r@{Z5W9%#v8Ns=R8xQL73LI#tl9pB$MoOg_?TA5}rKby5Wf4gSpWNx?Jx_0EC(uTejVZJIF*%q)lV zJS;HPQn9!+mu+f9BC&Oj0*}3*`Y|3>P5{Yh1~3|=p;2W7@AT|68|I1{VPD@cFVi8p z%^{r%8*)rRJog<(GBisA^NIIRB&}C%OwSIT3_Bc8;D&=-(?f&@LqVA4aU-~^q}k*! z`O{}8yn$YJsbl&9WyhY`coa#40Q-ZFlC_#e*VD^BHTxPc3N#290Sw)-6o`|zc+*A8 zqp-0M`hy9LCFv@@<@jU-?_NU&&k6R$ay3>;g_B6!cu!pMEr<9%au{+esb@a&FJ~wk z5X$=aF1;nGi$4;dUE)7Qlby!@GVX^c|Bkp;`idC-dl-x%dkkPB&r1`C$&Ob(?4M`o8p!d6*w0=OFBA!B$A_*sn{>PU zVB!-Ob)6uJT>f0Cj);q1ey(8GOQIe3_P`6i^3s!#>~^97 zbyQD1V(@dN*(kaBBXpP`Vl#C+M16rpDw(;b42jT{#+aFpw>(b0%3-Gpj4Y6b+uHfB z;8%t{{HFj+dD%>{Kk$&S@x=E8*P847*j(@8?)ddic=szWiVbQ||qKD%J(Zhj754tgn9xQs84?WOmQ~(=BXhg#M?t3&kyMfr_pX&DR z8KN~>*U*wn|z{RZC#UsxMk60DQwt$r8 z;esgT$f6Y8m_;cTrOby?c6WC~|Y`d*JUtG_CQ6M+O3W@K??DZ$vY^{oQQKUq7AI z{`KXw{`b9-Z*VMBJOGOST{sPOnuVVMpjtMqr-5|E*tO{`wR!VC*D zz=psKvvArxnu&%58oIHK0~*%JWIKSySe=B4B<4^{$k#ttsz9kGcthKV8mx8#SlD1; zgIbFl02{K-sWZZcwuKG4F$)_kY?v7~EXSu)DkUD(2wltxXNc!=*?E|eos%+oHCuVv ziCq`@m(l|}OG{Z%wb4oXC=&&zdjf9;1sj-N0r@*4cvo~4`vq4^B8UaAN$CQlNXMEx#YQ`?h>-kv;k?*1q ziQT#H5o2SJj`Y8z)}1M^98|_h46&A)9P^LCVfV zy@v;&I|06ZDsnQdEaX_*Vb6{(34gUIokbl?u&;f(0$Mb5QkUvn6I z4^}&y$VqKnSI3Ra)5UvpyayMEynA>z?ue92Mj?vTq?ZRjhgNdI!yXy@04d4`8w*jB z?sqI}a+i$(O=QZXuXM;~ZOy&Rt@``?v_lUMie#*~{tn$BFI^bun+0#{1aDa<)p@~N z%Yrux-sS>tjRtrDwlAO96Gwq+R#rJFDd@L?DN299g3GDEW$fc?@#V7c<+@Ru#g-OZ zt_54>T#V)gm+kT3lF$Ab{g|yxqHnJddxm&0WS509)`c_1m89}oL)U_rS)@mv&&Jz@&X-OMJPPKEy!n&Xf2pc_FK`-2>y6nNVJ z+ju0(__rJ5z-3$&)$o`?j>Oif)+vIaGt6COFqGR&7N`w!Tbtg&ARYDy3|*S{WZ}m$ zQMhnhroVKik0(uZF7UHpru7LbLFOb#mwA+9AU=~bR{8EbaMmLc0~fO7CwLzh2L0@R z_VZT(rT~JUkv9U)5QYr6oEb@gTrOhu^K1wie=;DqD3rH8C8uNXOMpViNcd%U$XGy6 z_xA^w4WnM&A*21Iwr`E7=ZdcF)xlfwjUJ@P1;htzh(Huj79uzTpRS`Gsu}Nz!^5Nx zI%G78e9Z3mhsYb%#0cp=3lZ8MLG16V3x#d(nN;CVGl+dTcHT2%`0-cjoW?)g1GU(y z;z3Q;z$5n*{1n1I^dW#A@dp&U2tYs)1Ee3{a$^PoP833xd^1agsp*}0Zw~|^`2#sj zbofUSf)NQ3z`m3$nE0IjLvmZNjDU;#eH2pA56N1lr3wNML5jc)Mz?}2uYNxJ@Y9=j zzx-4mxtx&vTD~x@ng|i4iClo?0{f&3<;rgs_B{Q_9tcrDC~x(?oPJPnAtxatz!4i# z6ruup=7;l1Kjad}=#c7#=n=mYck8BAzuQOt{+{Sfr2uP z@jx}dxeMn(xv}pkxR5L9-a3?**}bO(LU=7&N=uZYlJD7q@8$*>tBX>5zxHM<(DMOc zx;35`lqWZZggR=NvO9)huOWlrGA*VI@9Y90mfGe&?Zj7&g8GTrF08I@lpSxTt`5*$ z1{xZoH||IZgC+HYQ&q@|YmlcakSDJN*tr&ZBlWV>8>Ya7?sM#4WPBpayxl2w)qy!xilowRM|*fn9Wpq zlsQaKiebpt=AXh)nP^GY;{un9nq0Z9@v#hxt@QP+HZ0O;v@3k;xiIbeh~1L#y6bp+ zF?R=H;B-UeAbf*dY0{AH^bdcK-pkJwaiHZd&f!mQMy5?NsqxN$k0I>$u>;=vVAtG& z_7=CbO(p{6zikKseu>z06xb9muAeT>UxD->tg&Zx#-1MnT5Lkey)%~)B^xZcbW6{2 zy18k#o=t>PYcM$&FPe>r*<}1wmuVfufTq&B7m>Er?A=6-QF%w8`>8YflGz?pZ+uHP zPzgd3ppapd@m@<-hs5CEu~A?)7-Ak2IKfxVqCkrRH2@OWXLRE@UAHGej`U=~z63r? z0yD3F0GIerG&_b{CxTI(wy0$>7xZ)vAFwW=Z^4NLC;I;Sl|4DX^m*j`ftB-C&QB-j zl|Cc9L1C9N7&7E4okn~Ii&W1X5Xk3(Uj28D&OcYG;u$WJUVg6hu#3CWpQRv#P6q3l zUmHT-^;Yw>dC`dY5V#br(v|3y<|Fqz1Sxtna?kv=dC4t41WclDmnQ4XVfY5=^{ys+VdAVtK?Ns*kOYK6k9Hk08etX#nbjG- z&ebQuEB(b4oBnb`g+buqM;z#x{NC)SXt81TA@&||ls}5kO*hC_vkkI%d;#)Z><^Yb zj&s@UbOU+4AsF@o_8HJauo%{2*p29?#jwu-!)8sI^I+IRi(xH>odLs6P{1j5SLDO+ zex+Vq7w50Q1!QwrKBvWY6O@1HkxFHsLtOONm!B)Y{A1-~=b%r9j7>{}P0{-4nX_1F zL9Em$!AfMc9%~CBrRjX`EQ-?2SQPd67UOwf)RBcz7Di2mQA11_8H8}u{dWX?hCQ^~ zsGpovL0Y3E5+B_{uiK~hzSDK1Py{^1x%s7Vng}qez{}-EoxzxU_6~^eFCAjdLWLf5 zu95F5E$Ny_@%o2ZD6uI(OC*E3jeWOJN!@zZP-$Z9`-0K#HoX1m%6%K&K4(9zc;8}OhK*Xc68dRG zt*3)Hq4sM)Y_i_^9yYk>Ts#1K(U9v6kFj1> zRP}}{w`>@-4Wpjm9c7V>MKU%B+CrHPgECtg1pSPoo;Mr_*)Zp&Ln0gIyvd-)hB@0X z=cf?nTpr$>3v52Hb>`>!nJNDJ`k2-nG75-~d`73P1V17EyK!{b%*1~m93I&E?~n3h zqJKw-vMI{}RDna>`BTF_Jn+}QecP&f zpl9cA!KEM;xRAuMmW@?fHa6slY>)Cb4XnOztD4)$?)b8WHk%4!N@S zA7Xuh&H>{PpM}KpP*@v59}ZCXxl#pO^u7FCaZ%ur`w{Y4t_G%jW`0ve+%kC0JGPRl z0E-R&hIr8k(PfYPH`(kg!=BOCd9h)KDDbcYDO&oRU&xj((M)qbql5HfgI6(n%O3fq zY<8Amzh1+zq2rKf`Mj=1eg$QpRlL*Ln#h$HA9LU?L?hy_fZ<<@ zhG*pWpB{DLGssXs@>VeTOV#v@!~b~=dp$?m@y(U&`TPs-rOpt!k%!y|hxjh$ zg8k*^iuk}HsY+Byln0h`&x0)eLn1pEQ^ z@4DC@gox5^qE_h28$n1Mb;C!Fu-!cf?TSz{Ko|RioNi$Ot%7K9B}Rh<#(4`dTk1G} zmkqAOO0dW{!)1>%oMD_Z8U!4KyNVb z{m193v9e)3R%5kkVLeL;=Gk!Dl?~sq;X51L&m3#meEa&kjmlXKGOYn#?RndLur;5K(oOSGfR(?*~q|{C5FRnX8g-Cqh2;O%w@U3 zEgK%qvXU4U8?*wR@nPS(s@k*u=A89yEQ-8ikIhf`*lc4_Y%Gcu@aX|r0dEDoalwX| zSoyxS@_jiWCJP3YY*G}-f^j386ezM_XvkK^f-D;Sv5ny#3kG=Dhz{_KA-9d_*nHG& zBRXtE$CHfcm?n~Ani!5nekR6$h)|L^0)+ona}k5sn>zesLi|Ukanx*O<3A4Cw*Jeb z{Mh)96~=$GgnHxuXYb9s+qTuc(et-H1vZ`i#?CP*$z!6mbMA4Iw9VQXd~A33?zic6 zArg{MQv^$Zc2w8rv)_f0;H24O$gb5`B!&mmGvUDxYqVf=e_i1yAXij;;aY}#{l(b^I{x|f7w4J& zyT(2i(J159&X|lJ!x@t|$3KCe7!L8D%?+=&xf`Vxb{gUJn4M17X^i7$OX}gAFKJGc zG-^zv#;B)$lZNXoA%VfZIgQ0t8Cx9zpHBe19G>$vEEJectA$tdWWFwf&0!riXLd( zXJAFmqj8@-DYYak=K4B7m$?;esm_rCqPQR8k&jr%U9_pcZ=3S6VW$?x~Cz?Js3BE+sp!c{FF;Y)Oml0@qXnVTsje_bLr@?QKE5dQ)t-iyBmA&epvdN2O+ zVTk!Z{lt4Q%_lTVz2It$p-rnY5n|^t6oO8Z7;qYip)$h$-~RWx_X1PMVC=p4%X{&! z<4@w>V~Wy%h(B4%sm0OId+~gE5|>va1?AwKApBMp%HnwH&U@fPD#&|;BPuq87u9W- z8c>$Nh>kIP$SahjtWVPD5w6fP)w>HU(FM^%>X&GckobWPs9B!@ranvpy;2{=nCdn9 zHX&SU&NHS9f}>1Xr4M*e%_=p6`Y49eEYStVb3vr@DdYLD|?Uu!<7fs(DYKBN?#$;FXV#Kpf2K4=X-=qC~& zu=!R1Ys#@L;(}Vpn`KbY-K9Z6m%#s=ANW%}v+zH=o&5s-=T7(FpuzvVi>JZ=e5m-J zjR$P~9KuhCP#Ej{&N9QbPhJ{tw_7QI zn9m00^ZxaZ#);M_CyjDqemD4?pwUel{LV$;E}sHs=aW{K8i>w3!Hu%?EhwPXqRlOEiF`(q&Cb>q_)oC;5P>htl&*SkUV`hbuKCRv(Ywd$T4hl06a``Z zV>-?Gw?Ld)&5+&~>g5sxdQae&>l@Jm2Id@*cdISaJBCPHWsTlB{N?)BsX|Sx8Px@V zFK4aPhd-?M0f2c6%XIpPE&zQwYeXM~JiSXOs9PA!X=1bizx=kmJq`F}=r(C{jq_fY@jF`D9Q$kQs%fXC`$RU z3W0Ks(38#c0G^rnpD{{={Ed8M|Lq2-pALsyWt|BJIu-x3+wJb{b_)2P`#XEP4gTj{ zJhz1Znb#)Df;6X1C_Xckc%jvb#6y4$74C#$<3c>fOn53f1vQiTqoZh2me@s=t)ia_ zxTN{6SP8y#fT<6|eL{Z z{g>Sp{Fg5a{>xl++S;&3?+<)5!NU-qfMACE6S7C`JrI}*+_85sTHg`JPG#!h}Rz@%iLX-&eWD7{Q zUkE8B=RS#8LPGvAK|YCm93qLDszFt0I+plhU!wMxXdNZ`B#oFXb7?=@^|ejG&CR-x zJl43>lve+vtXrc^fp=e#pYl}#&hQvfz^5Dd04W3e;3JMwhA7xPBQe| zTb)O3?Y^^T&qk1r{)vRuN&3GdpY^)CuL@@g=W_x~ZP1;#5v7F9we*g^&Dh>gneu9Od$*ui+(I{Rnqfy-7mT03W-n{^B6ORf<`-$ zpDV@u;mbNHKd#y2E6%3dQa}!0HTuV+b6j)}me4bHJB^;v=owE~&zM(zsFLeKd6-`} zs4C)VC zYsBk)i`VY%qVjcjX9@W_BO-tg#SuMIVIQU7;{~Pf2nXohIA#@5EaKb$MG0wyb|bVO zjnLK}dmL}Vp-@P)-oA`h^ z1oF;e;Q_a#7VI>;{r9gGyjoN#*xO%1DQIl&Mtpzk;(LnV79#X-ufI4uoz53$JNwmL z=?;SjF1f#lI79)kH{Tl}KEL?f+o|=Y-=PrS+i5)K_b$X2_MGo8<2j!J^}k-rzLD{b zjDOfNetJ;d#Y=h~C*DgT+c~>homO_^Cbnxyt&|%H+PA!yhe2Z zbwc~#VCR8u#2*~o+J;4=PTZP0aj?urK@;fz==Fu>=-~R*1wZ5qb)EV`Rf$hoVR$3j z+h`2yjoI(kZW-Kr(0;esM|gZX!{IV`1KlQS|IsTAjgw&xt)T<~AU6zr=#NlvE;-){ zg8KqTQc^3NEb;d+A!#iBin$!WTk-uyv9b>O`b$c;e$76dY=lE~YZ95xafM}C4JtpMDs5!Ju! zh{N%#$m;?fH#gqHK;W~acjhABXbt8u#fgFCdOcdQtkr(b_@cy(W#o5yP`i;JwPjO8oLM)G>!DEaxg&-dNMC$@$y9EwZtu8`*r9vbnodMA(c|X^m`dWb@OI&4}{; zJ*OzSG;%h71^6n0RK!eQpvis3e~HUh_0`-A1zZm~L1V%Yc#HWI1*ftD8ejB0=IN_? z;I{zzVz%6G(Dtha67DI*h;_S51&K9hz#8HGl!fQmxhAa<^j02gdlrcTh9T%w8$BmtWp@B++V$4Wl*nF<7q zhULX}7uT}o)QKaX;Y$SGkSIca6K5`;*)qHP>ec|-7Ev8`I=8lM->44vr8?{`69Bin z->43a>hKM#4)S8PDH?&FiMU^$ouBjc=USq#abAmn&`Pk-Z%2T&X)hyh5bbgp3N3MUt*@E< z%2qxA&L1gCCiMzWf(D$Kcshyv3gYz2t{M3PmY93yixhlHExkBjTltIDO9H({jAK%E6$PRSJB;t}7N5l32 zhlsY-hjvT?^TUlv;QAzD5JxDX-tUw|bJWr4bUFund*a_tr&IWMue;Of{6}~1;BddQ zcesDp{g2MhUZ;EbAE0vw?f;LIGMM~FXXUzzje8^yU!V?*BEn!^GoHeS`za3D`>4Q9 zaw?(Yy5NTd4qya<>)H$kMd}8Ng76+yni1G1kuV;GqBrcr&CWL1>2|ll&i?*3=-LXL zVVy@=bHM)rM_jwlvBQju(NUkkB&cIzTstjL>kK2rg!;w}3;BktF=s}55)wk0W-sO6 z3?)#-k#!HcFLw_1ce;DKN=Sw{y0AoI3?md~no6A=owY`u(hw{1=7u+GWB2gYVX$A; zyVh=T=X89_Qbu9$hC~BA%u^bm0ZcRn9Zd*D`&YU|V zWLd@LL=pxT@xd+VB(EDBb8i0LN9Xn_S=u$KC|hWJ7Ib*eyLf_x@YVWo7SXK{wCZAd zD+#+Dm-ejT5-1c8XbS62QMeL|JBHW6Vjfzjn038Sue`e@#(VE8`|c=OdB7H1+uuJB zkH9C{(cmnBg8}v%cfc*V1LC-G1~krq#u=c0SKdFmgfHO!faMxrz!UQY2#@Xk8!@*; z#$)ya-1ErXHXZlo19%9E=lk~OM(MmIrSrQ+>1>qFM(H%at3qm)Q9ADr6R}Y`ADPlw zP;CbE{3^j*i&#RJxXW1GpE;FxayDW>&l3cL2j3$n2+r%=dnh{ATTB}Tt5L9?zJg`& zA2NUeQ$h(7LP8qf5hlcK!amGGc9bjqR3Zf zAYK@8<5cyADD+(NmCcK^kIrKfoa1Ofu5Y%1$-7nES39%%s)FePa=pbaOLtb;Z*#(9dYFY|)FSkH9)m)*m|o&5s-OZVV#zrlaGi>JYVdF=Qv zjl;5md}$n(Pt0L?-;ghB!FYLEm@W?&#AP+qmM4YM@*rVZR>EOmseNRI|Qr7@fu@RYTHr`(1j*UNNH4+MpaS28(5S^ zSGjLpTx}f~f=zokXUL83?UhU)%yO z0Ext(pia>&mJj>{L570!{^a}uM*-JF`M-rvJPZG%m4+|Q&04S}z^3ARYe2`X>-b8T ziS_jODrge56g-7NmY5VXM|Xj^WPw^_u-yic{J!OmE_X47D2G>T25*fbCtjb`(}G@CVHXw2v9Su=dbQ}FkcE+$^9 z7k*uk9qSq1_15}0#poy%&uMlcV#glb*D`1c4~iQBR(%1YhYf!Ew@br^U^qN8mA3t&|Vs!%>(q= z%ysN_y{6t8s(c)((LMWX?r><=S?A*z+ZB8@4?*c|PHi`*wi|_4erOcl2GXWcc%O>G zyBem=Q&3F=Sk7%9(pd~Q=P7jS34*hon-4|LYoK`=CGVS1@)~HK2Ab!tpn1d}=Lw=| z$m&pG^j(FSL>nd~jn5N29I;K4tR%lmQfmwNB#qeS^||e{N3PxH9H6T<2`}luN4FAAvwI4 zhM>=(eyzTZFs+jp!~6Qi0ltJxh`f;LZBe_O+t)~p;wfxmtQ*O*!@W$eG;COx0 zz+FoZ;!hMMa~wWWe>HerPhcal!RvbPdx_hP+4lg{|u1>j{8#-HI~W zAap%hU8X_kYE+pM{*V*F7smxnrMebelH-UH3k;`BR^3bfh=fTn}ud z3*cO9bhtk~Sgw^jS%O8);SgUNtNeyKxd6#k)5{A0aV;*88ZfRqAMELTtHI#!ok_qa7wjKq>)Rw?s za!DAtg2z#hj_T@|j{nu^cK3G+_+JN|!@~yu>n@&+zqR`~YWE=>wOSN0;3E0Y*H&u- zL?l2i`Wlmjp`^uSjn6_LrveBd8k0yQ4grkg*O}qq<}z+4C`6E=*X79uTo(@`k{~xC z%pH(43SJ8{vb6!M;$Q&dDhf~>lF1lFj8@Q~GA&s+HEzbp^Mzgl0_O5vY<#Crgvpa<03>U73f>H%4{wai?JHddWc zi7Z$4rCy^IA_>O$U%G{tJ1rQ;=I4&r?Rec*fTRW|-o57>KMX5`8(;(sQG^o65P(qt zxbUV3_+d&JO56}#q7dk6M}*T=F_xLgsNx^33pBYRNkDroSCz&8Fn!W;Eur6XK~`F8 z46(8iMP@XHIP3ui6a0sV`w1N3@qdrg5K%HqVT_y6b(I; znv`SyB1;v(OC0=n=WF15V`LHqgav*}{#`L1(%mUYT)cO<0omhnI>f$Mk}WfP_r30c zw|5I7bRqsPcUKk`05kA^JKe(~{_oDteuMveC(j0WD`QK599%FUNyWtWTG_%p^%KY^ zs!;0i7!DDa90?j?%906RBL0P7W8!U0D4$pp!^YZUoFpVcK?OgnVsQX4ION<-@-qPF z2=;)(d44RSBtq;@df@%w6Jf^*q9|f5N!D>1hNs9+5Tjy4aKlI^M|^jAewT_APOCKx zNgswi$&pzoS_ME%

=61q6B-8hnot1dh-UEnya762RIjDQgeW8jl<3>og@5VHy3 zmNOu1gc2av8n_x^e*{vBfF!~z((j4lj>3>!0fa?X7y+0ha018xT+3#~Cc_OF!60XjU&OkOY|lyQP}f%zfb0PhEY8@!Cjjc33{n1U-D zhRPbOlB|#d2u_tXc8Zt~uWZB(Xa-vwGJ78Zm@+bk4Erz)Cw!JC0SW;06GAEYhbLW< zUV?;pp#KG85>Sv*ZpZS{1S+?Sqt=EfDZtd{?VErBf-Fszkk=cEA{4lS8ZMzXz!`5c zrhG1;uQ5t6w}4@&tI@6FefojuUy9sgzo zqal*rX6$Z0hap453Ah@e2-rn(3Ij|%-iF_0@YT@1cBw`5( zLzK{VR%d%dS1S5~lOy>jrocHNVThyQPjLVla`?!{C;|U8)#=2Plk_$fJ_rbuwzmzQ z=kh-XK8!$!U?Aq(6#+>aMWSuWaF^T~0_;yfip;Q0xXu5ekJ{F`SP zb3JefV}K(u5jjPvsP|O0uLpEqJ^-K6aUUhTbP@(&gehn0oUjCOy^Q|_#VX@n)`$K@ zD-$hN2M8&{`${dNZl{%?OU$*LA7z;z@EEfm|Ipe1A24OSP8bB3YeX;veux1~hN*PC zFaiLZKxDss5irT8UWN475M6}6!f_m;9&lYb&)ngkICfFw6FynCf$N53=nim*wgF;3 z?+o5Qe7ePiQrr*ubRJUCiG2iwTLbV}zJwaw4@tj0hFq^!lP|=sX_7Y%TIefF;88M^ zT&zV=a*2H*kYbXs9(dV#*%6FQ_b~4Usj0s|7A@u@a0QbfFM7Bz6-wqCotP`OBtXZa zSY9Td``^KkPp{M@|cDs@g|Je5s^ zd?_O?4f!vg7k$N)@iy@fzy3KCvb&`&vU7Gja`H>&Z_7dwJNMDQ$q90^{{ z2*7MRtB&_EUt3ZINhx}#Dn2265l={(xSAH1=W6K&tI-jC0bFpBMxv*AXTAwjh7uTz zaRlN7Us4bu#3if?K7ab~Z`JL10F2cxq+H0y5;3re2SrK@8{-mD5RSP(=0cn56(-V= z1c-kb0ldk8k*yZsDXm2I02~DY=jLNfskp>~FX%U7YMr0Lm}e7Cw!8q zDz-eO1tjQ?6j4Tn(!-}a3IuF@IvmO6FXy#b24Z>SGB8dxulm#zZhXE-k2#eUnvv2> zrBO25zRFrGGDush<8}8ui365dQ`77zK1fT*VpX}QyCjKONL?StBb2x_#SGC3F)aA^ z3WB)LJLS)Ip(i#3Af}#LnVbb+V*{MV$j5^TpvX@XuC-Fcn6Q+jvh_yK8<|{~7Sxy$ za;jKeq~h+R98QoBTARlt*fJIH0?}6{i#wU2r!q({gla0g*^Noynwi|zMT%GGD_2$# zbJ+;F#V_uFBreRdazUW~KFt%RXBrH+5^zEY`yPi#TE14$r%(=#6MTt7G(_*H4?`%u zj+wda!&uq^7*TCi2MIA2whN9vd}y_D3_~4l*#JLtR+b9mC_&s_MNt4EA{ahZgQk^n zlV7e}!qwai6>1RApvm2SqVjyTaQBXR#s$^ zMu2dRbBPlY$#o!@Yls;Okt*GgvkhqKk2nb*afJE!dR%NxND2Z1u3(gFjg>8pWQ&;~ zZ$Nlwb9+pdFHV+h0~q|CQjucJtB4R+)L=5%mJ%0@$)z-*{4@zCpr1g|Lj#5q@I34C z=bkBcjF}p!4FLQ&<$8H%tY1iY*&aC9`-n|TC0i?ID&^VT`H0nuyCh*cMjyFqKoz&6 zV9bAV$QHwR_TdyLN4+nE0PlkfG~p@j8cnP?muaHbW5`B5 z&_>K}vrtbE?Eo&Y64y?l*NB$oir*hewMWWL$zCv`+-8-I9B`UT$IUqXd@w&+%z zAklQaBq@zuU39Xd~vz<#FVbyZ6CdP55wLYB-3b&r}@Fn-^fm)1KlfD)yIUipWm;i6Y=bu^smFDR}qKQ}F)iqtIHVY|!Fesks$R;lz9Jih6Ji z|0IzRQRsrUc_&iV=3Hl(;sCW32XIx9t}0R}yWWV6!zxoLQ>^$%D2+@m#0uACF}rwY zIkIQo4e9bE+&QvyOfzePyPor1AupE>We5NpM#6UdODN+*~9Jok2JTIEUxV zjw@}hu(6P2Q=AJ&#z!H)1YGTk`S7q2WSOJQ_fgDHpxE4o)Pu4*xFM{oRy5|ShFOh# zIh1QvN7m-3m1V`-wH{AxoHl~1O4lAC(Hc*h?Uyjb!UD(+9RRhnM8S@?bLe#h7l_am zN^<0hn&zS|6leV`>Li1>BN9Ksu{v$I9I?_Xbyw61Pi}!+F$D#gXLyWA$}**#ijA+ruvbwcqg}gQRU^}A4Vi1iC&pmdT9=wAkhr26Psn5 zRJ;=>aga?nR~)6!5OVnj6VZkX6iLdp%S5dokxQmJ*hg%x@)EA+k5N`&%hp5wCDnfA`LSI;3Uda;)8=$brOGB60RNRXVbzAXz8W5la(x1g%e5O799btUzhtL9_L zTo?r|GgY)(+20u>KO->-$#4Q_ER;5%M3f~w`^qF`Tlg`N|8tqD&6TyeR&AU-`SDbI zdy{?A`wjMm;!;C|K_FLDJ|LlO1>LvGTbZ*8ib=pVrK74nP|+1!pGzm-Dfeo~ur zA`}FR?Jsd~qL4&Kugtf~lQeRzAVhoB2mLlD@GYiED%i_+X)sjEtJs$rkD*IZ45iB` zR67P%jAd~$6QlZ5l86%!hiL+Z&vz{9%G#)jmaUwS6kG|eT6F%_)oJ%{#O1Y+~88t!Z*)MXox-a=e%!IJPHO`Rn@Tn$;_7 zo^X~F6SM9^e%7g`y`*kNf|_*J3pw+SE+Gzuo69Gm)TQbJl{c&jv@JG|p~-G-3ns`4 z6Hr@gBo!Q9#SMC`tTwJQy1DHnJdA|ZBDcHbXCWxZFjhYk68%p4@`pJrseWc>2gPq4 zaUg$C@b-MoH7xZbA|dNLN0O| zn@o|-Mfp=&io#ID`W*?>UolE3rVK?)Zln1j#N+Bu>MaQZqcQY_#$Z0kKXkCC{-A#I z^i~w6`Xq<&kCZSdU#X2H^|$cOsUOx6NcDmwaL6e(9VmRcGrC7M1~LZyksPzu26)G3 z4~~Xn63R@H)5VZVPoe8tpLCdn*7Y(IC`(l1pXfu14)%bERz!iE^r}>DR7xu|L9CY} zi>7f*5+;2Kx!@rY=d&$|H36Zrz>gvzl=Dtr#Dt`Y`jF@3LSq-9eib%$MP)S&I|>4@ zsiKy)M1+(b^~8wa&TSD1q_(YdF;Hql#>f;+DG}SGUKTebgBvmzsq7>qMTs4~mq`=R zYjXLE05@G{PCRkuk}VykWU(|8 z^;a#OH)V|f79a_jz;S4#okc)Sqg_X{voG50s8bxQGRnHO*rK*7$#t8X(~x!lZMQZu z?Laq=Pg}w4zY2{&OqQyGY$e5RM5bhm##kYvg;;cxrM0n<<z%YQo7q4eGS`%hLpeAaQ7T=(-~C>>KqC(DW13Sp?T9h2b5WU4yvB12~s z2asxeOd`3b$jerNIFc)?WZAg!RY3Z`BcFL$tY#KRgvK$OfK3r*%B(XhiB+LsfaVCW zl!|seTk71F)?B2IBbBRVn)k_t3n-xQW>0Bk;%W70<^F{%IqgK>#gW4Q%VnSr&Pmt5;6g(g!;@=MV-k{0h`=p4A(A)MFB~08euylNiY&c-D?)ahhj!I~~R1 z>>6-1y9OLBQiF`#7w3txy|;Y#j&X#?Fth?&v>Bj!K0?wp+D8C}_)@Nee;pqIc|J;n zKS0L#B1lh^>Vab&*sNw4&*gj`44iG?ypJ%$Fyw#g(^!s_e-7s-jtC)QK!EmqTUH2Uag|6!g^2>_-^u4Nb&g02$P~nPFjXQ=Oj8&IpN0)n_}S3TAi3 zI}J3ex6DhUw}3jZXiH}*mEEQtT12yq$G+C(EIvbgN;mkTMo$zX{e$@Kp-O<~R@4Ruap{mP5gU-xmliD7T@P&bn^Ufa) zZ0n@ztLz*u%XluY^*&PWQ*pBY`q~_p08YiOg+5Z@01w>Uc(amckmWKxz4SeGlG+Q2 z55wN|HPDC=+Kd)v>dj5tZt%oDuc(1VD2PLdMHmh@W#WV|(oY9m1w!(`u4?m->( zAv?URyg+P^80Fn35lzQg1^tN~7mJzjNf7ZKsvzh@&NxiX`SlPGsR;z}OjY`NfF zL}l=uvRD;)Fy$6FQ|FPGeQb0Sc|#9eVpu#yvL&0HbCSvP)|Ril*ZS?Z-&%<{BP(_+ z?6bMRaexlewvOTIos#xqbJLMU$@9AA*xnE^K?3h}6b0a>LE(KI&yw>0e1yW7tLH49 z4Fxt`{&)Af`-cVjzqi-jZ{+`7JR3kkZi}U>{MC+(v^-aC!gGCk&H?VtO_8%I#3}`z zSP6(_vc(CG*Z??xr)_7c0yu9e{+**do5uyJ0pQ7=@hNP+fZl7csY6BT8|`gbCE!7^ z5cy4hWeFairh@_g3Y@l6ol?*)@F}c}h0e%)q1;*w(ojb-5{m2+TQAzh zVG>^@O0-qF94S|xa&c@pPNxN=cdV!K@)u|@KtAgM=My7HNsL~PIYaYDs0T!^i;3{=YpgNH?DN5b(R(eGF6)&Vg){3J zJ%2n22rQyd^#tZA%@*ljfF)_5LW?=W6=n& z^)r&Ci}Z}w!5c(DANeq)2v9Q487ZRa;)+eFu!#*pJ!LTkoAT%uc>C#;rxh_2G4i*( zR<@5X3o)^1q6gJk=|Y^F*pTJKD{ANI0F(rHo;uIujilOj-n;K*Ygjs zMW^W$S&DZedrW3FwC3I}OkgHI+`m(&P@1fFktsKIZRx34Tr~y#Tpa|OYkc!~0H5kc zI-kr_Av&3~FvF9{uIXy~R$bAeUB!+rRH84Rb>=Fhwl^gNV|B2NG!wJEVj+^m9XW26 z$cEb&Xh!3{2>wwZ(erXKWj5yox;?dXQ5%&ax-M04zLv~j%DUEy$_LdOS*C#=t|J2jUQN4!mFBH2W%&#IkrI|$(q$fHBkjr0 zfRTU0aX2MKU0F7^9sjI;=Lh3cfy?ColvC@p0OB_vPtJE|6+6x!lCN%Vbt=UXDn~)? zUtM2M9c^o3)L-DSa8H3##0(fIu;CGC1%?19^<)0KCkv3_3)!+4ek;$y)!^0VU^Y0_ z6_m=17BmL`kVDFtnlS^d;O}h+i`m%6_}gHGukru;;W|o!8dQci}vo?x6p!q zU}H1S(!cNRj^-&mOYZ+kfEfFn;H3UqpqcSMo#OuAVdr4C+5fwXN3lg11(|;<2Zy9j zhHauhYEme`BuCB(XG_|uSp=*NWYkhouoZAdb|h>;FxyABh00#IwGM!?J#Odu3q&{y zP{hDqN#(agZdJuo%<<4TUH$_3B8^qCKrBTo2dueZ&`*n7aPlOH3`5ByGZm1M=4o8n zvt$~1AgRs3o6H#1&m09iHSMEU2`vR~ZVbi6VO^kQ@!CzKU^16ht$Y&pz}cH)d)#vVD1G|LtWHxxWtCa_ z+~FtimvI5T?7ZyGPfjM-8@c|-pJnC03@NZK@tATe;H>*!cXo>RzaAXyHTVDB$y33_ z#o>9i>6m+$zpzBu-L{+;X<*6(dWDd@8Q`Io`&stydiqi(7L2Tc?C)-v&K> z{qJ_VrThQ)ySsbM`oD|EUeYj*Dc9AC)7-jN)iyYl4v}4Ko6}e1*SFcxZENPO$|xQo zH<-laz+%d(Ji~|s;`XA zQC&_@J!H9Uz!T=yYQJ0*oFKusHuIYKC z;VjvVd0WU<9Nu@6>hdN&Nrxo$yMFE7*rkWyv zqskDmk+SPD8C>5<#lb1TJe!Oa@Xl68)@Cf!Ctt{TCB2QG2Wc<~3kVEWb}QPZ{ z(XTAznHn*2X&t2`-F^p)=MByNGMlG>Ai$_(QL}I z?xrbMb$WdC<`(Lc7<+c-T>Go zDhyX8RaM}@HKbYMY8ri&C9X*1j82%Rz;po%WH1CaC+^Rn=B4R9nZGrXxjANF2T4iOw7zE zZ)2j&LoFWUN-vDQ1uf2*P_946DP=!J_>ZM0UKtv-zWUq*m%%x+C0Bf0WQPjNJ?J@g8~DwU@nNqMFeh?})(T`XdD7uBnAwUo@+1B-Hy zDW26LEe0M_rgd#csJI;cwZzyfdrRAvyKoNLNRlnc^32`JK29OtN@YU>=9pLxGjQ3Z zlilf6SSTQeKA2xqR-(e?%(pOdHn?ex6T6Dynz395={W?*%418p6jiqTywlwr<4|y) zwoNL7L}T-@lyBxSC>82mWfxg-Y1Ir-0PN<1TI?7d1sEyq(&HK7I=0nm6{x+K&WN$t znTHou&YvrPoJg7r@PBaaGeE zP{1)PrB^EF`8Iea@v6A$S=}y0xo4GEtjaRyRdMDRG{`*)w8Ba!?;0*7tZ7j!Im8IT zkd6M#e=>8V#>NB|isSV)NaGkKGa4+D=L@Mseyp7m6{+*Z74RZ!3(BR-7g2_ntSzOO zG+$5=k+Qa!d_udBGTx?srmmxWwkNcZ3JrU3D`|~OsHUmQtf4|tD=eZyNz1OHLNQA( zqe40L>!@UIYY_6>TGQxt6?#mT!zeM`4w!y~z2@psyUx;eu@&ds>x!u-#BTL$gASCO z$OX(oSAq1(XS(~dP{mAVXkGfHU8AB$kt^XcT^IxjqV#nwUS+M$#mh=#$`S;}uXzXb z+HJ$+dgVpk&Kvxq-wc4GH2Pk?eEBkOh!st)V#mt_Z7W^g^r|*|>Ibr9CzLIPnOREP zI{6_ingscb6&pHo_U4lKv7Rcf0vhqh2D ze#n6kqw#wj} z&CHF-_mnvLoF`Bg;_pSsQs^RcYn!_(X)4&vGN;S)WH6`y|aQ#;e*MC@CfA)*q zdi5)UY!(Wq$^7>@7i$hzM}4Tn-7&=$zsw(PR)zpt0D!25?yY$O%F6qHg8`rI7PHZB zDkCp->ba_lNPx`I+8pnyM343AqrL1+RIAWx$9yiuT!dOcy0vuXz;@>&)kgoVq|{EZ z%-}H+laLH2r!nW0Z%9O00&z}w)h4ML-um(uq3%}p#^_eQ(2}ulZJX(9t*Se%OwzoY zW%{mQTZG7W%Y?eRr8;H}_w+cq1vDq?nlHO8OPxFs^(@qk=_h51jhSmxaGedBS>deq z>HF8F>>4hxpqgXi5A<2`{ExhA&Rmc;{SeTs^MBoLr+EImv)^sb|J=oMeXTZha?!8m zQbWFIUDs{t@GZ>;RD@Oo0P)zLCD(r@^MFSez|GC`M~7mtRZ$tCh2Z#E-QrQm32;lwm3a7&`V6&+0ZK@@;x zvaXE{@D@cA0K-t>xq<#fJI5<9-lhVusRE{Vt%a91l}eRPTaGg;*XgTkw}L)5Hh{c- zMCTxp*$oie3;7<%p}5V)aI?3OGpo0ZoHlc^C8W#+$dr~=}lGz$!UJD>q;dpggu z{GSARhkdpUILrR)?v(t$`wjln-8>ccpN;>y$v_ z$+Zgm&MtT^om$BEF|_%nhT6!BE|?Us>dH`4&dQ zY!iP2d{2@-4gha*G?dq262icc7v!19&@qmBAUmovhF^Q&V1IvizXia6N_5>yG^t z<(>3B4w1YoTX*k+uwvi|Nj@xmnC=qui zYD{$61HU|%!}R>u9NuRk4f@JZB!*U}%di`Dq_8L|Z%1In(*tmcl0NMTIrPi(lp^s` z=M*SB^7f78(a`UE;XRONJ^X+A9T2nl|IU8t{-?eDoreG4#q(|P|J%NFVi_^8`o$9~ z)Slf5Auc|uz7=8(_d=}YW{8Gsum4$+|6B3WvoC;{ssDG&@gF-){MVg4a|Y+%_RSCZ z`1^a`e!0`wlYEd_cE+>^{`yZ*bWRXxuZ~uRj~GjEKV`@|xV2LM>3yZ#fa}+~)}F@Z zrrC+Rw4Z%l*t@#?QHMr6tnw_&|7SXVtYKem4bs!Jc)pqUSQYT@pe3LEMRqk zKSKTmIC*#U_TxKo5B!)U2;gWylCe~OoyzmU!aA`66K&IlB`MkhoAy-|RrjoInfv4A zIfzXiX0E}Ip>xzsOl@qd|baceD*loz`26VA34A3Q#D!KjU-xY+jA3HNs%dK2?`Pt z{Z9IG*WXvAm5ERVElxrJ(U?RO%|B9^Ba)X6$QrjEtQTMH{#jtvN{VSZ*!ND|=47O7RF$U>JO29Gh5Tkfl@HLIA# zX&9&6(u{^zs6Qg)LUQ@kAU(`Ev4b(DeEKA4h{fey(>hFq3{0zb@m6FC)+!yDm9hi{ z*10A;KN)i^wL;lT5J!R)dxjUVc)c;#P$@$Zld(a52=RC&LaIw#owBtOwG2HZ2u$E? zUj)w1O_JRFs9tAPm%Z25VZL1XaW`bxZf+YLy!o<)XE#OHCJQ zs$5fLcfRDsDZ7Lr4(>zgm{IW(EfMSOO@*nXL4gIZFy{@ z00}g>*;B`&Ge@RyP6^nQlQb+wJ&vwz9Zk*`P%3k!X;`M$DdBigh_qw8b*5tpg^hYll=3`v+cp$ej zN-pvIUdC!(ca-y}%TwUf>MNcqF~LN&p+w6E5>UF1*qNhm9)0=a~L0XH{J%Ys*T%a>l1p*|MDLaBAo|*7EK`@Er_cbYVq2wLt^HNQ6RG>n zd>rI%Hk+?-btXmTF7{JB<@NU$t3wvNe?F1O3g)P2w@F*dTGh37YA1h0hn6vWmIKJy zYl89~__Lb+2c?F-V*K~cK@tC9|L|b9(f{t^sb-gWpMCG(q&YHa4AVz&U3cjk=)uTP3Ywy94bvWo%mT z7hnk<&%mLbr{ zcqgmedO53pE9tT%j7X$sTtIjZZ<0t1<~ z|JNz|fA^aEKkw$bF5~}I1|w9(1sAsk=M2}aa0ch5)8q3DD})LAK!S)Ja-125=){{L zV&$wb76=WiL!Qoe9?1|QPa;$KhdBGt@b9Pmtc(9oI}9?5|91}y_rLCTI-UK7|KG)P z5B$ICxW>~E11p^ISVtE$V6M%x=(8mMzeEYcegWd9&IXuu{-d*3!vEfF@W1covCsd_ zKPc}-f}?Cw81(Ogw5TjjiHUzzmGmTnNbKu+b8edI&P$L_#$yx( zD4|-L&)!r9B20@w(58TdQ77EaEzle zmFC?NRA%MPrc&;PDord8Wz5a$w!$->`wWIw)J~PL&jqgfq%U@1p!WI%Nt_W7XO#zx4izJNn8A1zZebM{D1l7bzc{o6@Ez>3B z_s7U37h3-kpxYY{qOJW71w{avH$Mq*{FhO{}F{_@P8(7AVS!4MtkB-^_@u~PB{&&t3wYLP2J+F`(*8%ZUac8XX z0G=1xDAi5TMBXW?&-D-#TaDG{UryeAcz1OA?)=ly$9KP0_CZC*gI@+A9MWIyrsWXS ze%;*2{oVq76p}0NJl7G5h?#w%(#_}&-s+zsJiXm%RNZpf5+RxrG$xmb0-lIZ*ngTS z$g7S?gd#=(q?Gs=GA(>~N_j1Tpbdp0gSaOb2~=P&*K|Y2%*LCP^E5vmCQu06y5@@n zXpO^e7dyKxGTrQ!Y=Dn=m6-~T^sa&Gf`Zq~|4M)#|XIG^*Uas)sp{eKM}2BwU;|yXB$w2B!BfsFdCwe3>Rmp?S4r5ZAG`c z<8|6^emwg0!@CckfAGeE9JMk0osZo}A{LNHR0?AQT7&8HvtqeX1Z08}G2f#Yyp*Aq)mQF%bz1i?UuoXAj-3}z%nnr_A9aAa~51|ogG zL&W5)5zD49%(`o6C6nrNlj6^IAy1nn2(`zMH?^(vw>6P2G`E;-W%vKcz-tj@tqM09|jl%l~{{nU&BU2yv8twZ5t;$v><6 zlepASZ>!T;+j_EWYd*VlLKzxYXVmJFWVU)S#3>zdu2lp*fpKhpcD%0FX$6S-2^KQE z2Tl>AAdLaEZtekOAT*8Rl#aL@RAP-Qz7V_>w+@{mkxHi!$W53_f3J0sUG^?IR`=`k zRwf;))@VyYzge1H>oOyJ$LkoDy2SdI%Etl>VEX#+?w9c&clY)i{D(VvHo#kTvR~2K*f${ue3LyCR(Z>(2yqS+%(Bi(*7B9ySVS>OVOfc8bTQR~;7i&Z4{zDX@L|E~{%*8xe15~IfmAbQD zgVL5l2nKz0cyMsA*FAXks=M>*)lSFv2M|KI+kr3l{e!~+df9Dfb!~j;x*xjxtyanY zkOwUlk7Xf}x>ii%UhaHS(33#Q;-%zVs)tS_KK`U9!-^^T!1<=kUC7(o07oKXKV2>zEAXlC#OT<~9O#yw50??mS_0UJ5tc`IL&^8r?Y>82@);}QXWIgp-8?6oS zJ=dF|KN98{j)29#^cj3w*8Jt|k4#!fSn^9hO%guNwgX2h%FiiKyi9H`AlljhFq&vi zON;FlcG4y0rYsZ|rm=j-FqU+Gj(FXC+CW+8w)h^{OG6q3ym5913@{1<-3U2O&e)F@mp0W=(qR3AZbb5igDu4%* za+;K)ygm6QK>;^FVMxsiH5!X-?JqKs!zx^;D{Jog#2ay_?mxckcHXQ+VfBTG@FQB$^I$yQ;;es9rS z5xthx+hOEm>^B|C<^J)r2^&uBJYMsB+ZUY)4UzB~S?x~(G7)Z?J3rQ;9 z2@#=In&d?Q zk>WY!ssWR|H%wrks-ae)GE|R8PtbZMxTr#M(K9fmvae$jry^{{>h$N-bTI`)1R;B- zB#Y6bBq0ePDdVk4L=2mll9KoEKdpEitpw{(983X448V{?Ly7|=#fad)+oCdg<1iM~ zqOURXIr|mli0!g`(oB6X=zyDSGxzVM2YVQAmv0U5vtr~Qbwtk{B-g`2&znR(nBledavEq zN}YRsZMenFjr7uZ*Vmje#T5GJ=EkF!zJ`SGLgK@)cYQ6>al%Bn&;~d)%zI1%VXPS$ zt~!&8HOauFUBqW|%G4MlE!V--2qnlOJVBJvZ8__&Fhw9l@KOwLl!jr;K29z)BGEs5 z$aq&IDca8Ii@dJ}B)Nh~AhZSJ12l|k#K+sRdy0_dB_z^L!l}| zb-vB&II9u8(Gn8j1MfawL#E#qR75ds(q}1nz1|?e5d3&{c6{10a*7L?4e*vIsjOP4 zJnAwLr%0A%6ZZd9DT~ufGfm{;0m&7^YXBsXHOVpuB=M0T^Hd%7whNy%F6ul9G;37h z$|~1ufc{AN_n2#=;;tyUb0pXS7=&=B>QU@o70n3mc%>8O%N6$?Wolu6!eaw>*5cQuo!(Hdqsa+cTtg$y4N8xa#dxdBcP45(_1n8sS6=cWkP z{*3qBWD#cEAdRFOA5*?06XXk36-RpYDPae+umdp0qRYo{!e@fqW)(Rg7^I8~8Ksoh zoNaH_s8;%QA^kCAqaJYDmoRB3Y1CHt$hGsxb9ofvw&)0FcJIi-IPpdcyU+e9p0ls) zp{1IW+jy8c1nRl~Ntt6_`_}X}F9ElsdcetbIe+AS77Q0-06NCw^z;J|3N~gFX%@;g zSFd&}?u8Q}l_#792tF=_NrX5HgvkV)eK-}jvCA!IG!f>eT-|yJ`Y_^SHy~*wp`UpA zw>VMah1bKUrs!m`nnlDI;DS-jj-u)!7PO~|k*bI=e`ri1p-geA)OHy)RZ1)fRp$ko zSgi!{9SPMRVl#@whz38S7C;&Ms>XeAab*yg-qOBPCJjO+om zE_6E$TUG3skB8UNlUQ#1ieV7)ROpp>0DjANnwq1(8MQA8ib+&`B8je3)|2lKsj3-! zP|MGsM@Zsr`y#qZvbl@NOfIoyNk<@`8&1ItXggP;5l#R zsZMPj9{@%|r0$j!ga7~s;>y5@u2T3edurQNhlQB!giPbM28_XB4_tP@g@3#P&?LWegT=PT+{~ zk>OV7pyUrWnmwl0tl{hYlf2{F%pm9|gwm}4wOLr6tEv!zf5j(ib>tRFvH_!bHr@(C+Nt)DUw5vEfd zlR!4)M{b6QxK)&K1#$i zQ5Q1n$RH`_Maf*si^_y@#N{8wT;OvSlxCq*nPZsibmAyZIMM7b!NU=&sE)TH+ilJ`KT)e`pG46~h$5|R!_%6u;f17&?@)5LIW8Cj!k_m`cQofZIdF)x*@f_;g_ zBP4IiHA-$EhT2TF_ZsKqU<=fhw_Yb;z&HnFQJR9Id1y+dTe#!uFpK%nBD+*3!{3<`$bE?H6BLJvSAQpPE)ZT%7u^! zLXkSkc8ezwG4JZN#XmU{<}y7b+#2__>y!E;6r`cvjvy4-CAMC)Ihl3`i?1&K>RP4C z{&kgcZHLKlQ%`vQdnzw~fd&KQvmS6hkyAwgf33X|J>b0iBZZ-p@2wg#v}Zdv#&Wmu z4H?H=KX?Ao^}2g*#{qe$fPZGYb>0t{>1{n;P9HH?xGx}!`7sA*dCIAYsz?0?xl5UE z5{%<;^3RI>!>nP?%(gY?y$0eYoBXxKCV_neT-BareP5X(JZtCuS$)6RohE2J=XH-! zG89{cR|#gwCOj)&1#V1RA+-hSdwZ;cOZ{wju89b-&=K4!NZr!@rb;Fg&PtV=7$(z+ zI*jqjM1i}xBg-=>mx0K>IQFM;`guYX1tw4P+qHn38+&`zI_#fq(f$SgNC^{3wQbR} z4mHd}k>1E~3WK$&qvU~ercaBjR}>&D+!q$ir`ngX?~9VkY6itCbV`=(K#HRr(y#=Z zrF=BYSgvMkt4bJ@GRia-<9F)Q+SgDQ@~O?Mzf_3=BH}$k&pkV>Ab&FSFnp5N4|^hD^O?mV|5?aw`rBWoQpUU z1!4&g$xu%|2pLkxBZa*2vOam1Qn`a`>pa!2U6X|(W^MY`XDn_HXC;((cIr#1NNW>c z?^~`dacVBBm6a^Bu6JiKS`S#AUcLvV_gcn1=@eP;Q~5UH(j)+7i5%OfuRu<};HrQJ zoNPK|GpBq5qtcA4s(QBWx}KT4R8-FxJ$>$ev&ourPfMP_a=i`|JHZZc)D0@mT)FH^ zTO6<{2y?8fTb!-Aim~o>ae$NEOfw%5)}1fT{GCzm+1CKZF&~|ZZ{=$1N0U$gn(>@D zT3Q>gZnRzUA|sZ|x5}913aTD8DG%l=VS$m$bOXIEW$0c}JW*hJ%)O)841=oL@j zmv1F=z-C3>%Ib<@#jUD9#!EWVGN4XTcnhdMkQHy`td>faCri**=GGLmRnhhNZn4NV7gS86+)m__Wv5dqd}lQ#D^#~<&XTLz zXRrFaV6KXLDqeswRn8RBIxAIF5uCKv*2oB;dD|mXJN0_W%B1BFEEmYi&%|8PX|u7I z&|#B!!C}dh#q%#vofXDprmkekwf*Q^pvfu}T2*=%XaXD^9JQ`Ob$PRSP`*2K``0l9 z)$G5zz9&5fGZ?t_rvhIrtJ3#y$@xnAe}&i-D$e_t4~EXd|2o{?FXI32bq|{Tzq@$E zd^Em^DSId06mI~WKx4lzkp9srhc_0#DPTG*#@sGHgW4>v-mEp&ZMf#B;%b+vTb?nE z?S|$NIiAM?diD9Q`EU>ZKG31Au|i%Ji6jz-2&Vnzc}h_(XWcz8)4iTWUBJHWq1wZ< zgu6tvJR*^fa{1}xgTNkX+dw0#albf2#GGI2HV@?V$j1O`KUy9wcy^8|ChWTI3e&Ov z{))SsmCM~UVXvu$XcKiD!Kso6 z60iGl&0u}CL7d=A9HOBeerZL+novvr?)~wbe4^v`Z?hLk8XeJ}C`weIVj7|zcz(j) z9z~PqMHGb6_Z7vRKG0AYT~+6d%!M-#{Vy%}3b&+R+vxH%jpI-_4Pltw zC1?8OV@jFmo9FVaLhqC3c3d#VQ4e&?QJyjU+5-ps`@8!s00vabm^BrbAQ*g(!pR9C z;wXy@`O3!ql7#6vyNKesJmIvq@ibREKNmjDw>WvOUx+CP*@(Y@^1^{1)hm6>8nV%` zfUXv!m+eM=LMSk2E{SSGb{qf3u(6tC|$2%d%V;~eX_w`3TqUi z6}$fHS6~1z>k561Ny1Rl1C9z6%3UBdch}jcA1XwSmC2mpC|@3ApbuJ-p)-H$6X0-V$ScDO0GJL zXecT)z4jD4J)2mI!pmkM$c7iB(yO)ePDdyMM<5q6v5cBM4T)QBY zPY#ozb(q}+t~>Z6h+aS2{PFYY+4=k9t=u;esN$wc`1;xA@yX}^_uuC~oqX8Jm0vO2 z^;J%VtE6@@yAz{RzP20zO`$#Ld6^DKy z6GzvassanWTG477>ZC}1N|77tkJH*Z)eln9pHtIUZ*tpo=@c%#z_EKN(kxBhEH{l^ zv2eM#+@g5B1k%OQBo^!mLro|@y>27swC9fT`y5U7*Q~Z0-fB&g#QC zDs_G6@?^OZYT8|Rg`!J)sY@=X*15$$#3_pJ6Ee0zE`Y?BGFQnJ-b<7c${WRFT!2(iA3D2&k1Am zQTTE|x6|nq`)N%!y=a!Oyg%}VLdzqs`7Y1s1~{=u9P=!4Oe<#N|Ign0Hn(jgf8vi1 zuVM~z)}l@XNm-U{R-7xX>}-7JBr3_-{cX8i<`5W)Sb+c*14`jWzPji4F2AqjSJOTJ z01Ws?qNQY4e2`e={OReL>FMeIO!qCeThUBtS$*@{?$6LQjcPUMFP-acGg9ccC#`Bk znB@!LA>XP7Q4MUcT9v4kU!_xY37^^mD75~#u~q^Lry23j87&}rnHB8?PSkt>_8b}a zL_X@qW#>0q5lkAAC%>J%J^S$a`*-g#Z_AfvL3s4{F>851RvHH>UZqiL{ zkgI#K$GXvY8a!(2%j&FTyd>R!RVEEP{X+iqCpiz_W2%laBS40>>G@FP*SI7DXOlI4 zwYPZ57Hc)wWX%#PsguseB0I@)i1eUKV=R~_^`1n-)|VfS&;IiEgS}?M+)=4fgWTLS zM;K0LSws#G50{p2C0(&kQeI;_5U#&-(_ODpFTaZF_TugQrVWlaW*2#Wyprq9ijG<< zIloQ~r;1hGcTTmVZagYGlU=c5wcY%C(U3zQstBOPL8+9JsoMxNn~wRG>P2k7Ezm&+qLPqVl+8=%cfoTkF`}{^lLd3v@LiP z6)9WuD-I5vhPF-h>MkD1%hc<7ftq7$RS%nL?iH`%`h+W&pgp}gboDUqzcm=5+?q1{ z4-gWn!HY-#?|po3=l`|OFpNS5xA|qS zPVWQ$+jzvj%7<8eC;!I>|2kgrKNRS7>vQqdA+HUJw}E(du;F(Acy&;HIx` zCm)_0YeOgPFK>gL)Sx|iH(8CL+zPDFmp2*$;Qg4OKCM`fc4D+N#`o^fa_HMTTnrm;oikTzc z3KQ27U$5KKyJZNRTmUNpaE@~K4Sz!tR)XJ}#a0ly0dmc$h5k{2YgKRjUA)f7mKCTq zyU^yCwOmK~x*KB6xu!P97u4&mfi75e`S?Q#ZLMf$g}T?t)sYnPa%eK8fQXAem_TGw*x3cTP4(4g&3RLC)donX39^N8 zTYom9Oh-?2j?&?4;`^i)uS_hScbxcfdPS4iiU%rr+aa6Q2vmnM)Qe-K>mo~@LDb?Y z=`&_m#XUWNC(Jsxc55H6zk%d7&Ael>hu1>9M>PZB+D# z7LKe4ezP4l;&<^(t5uOu33%7hBi4ZGCowNsB~#=Y`24li44Z(ao}scNED3U|`O~qM z%i7AvY^mO2}{Xt zl4Y0kocPP!gFf*sznU+O58v-}NFQ{>9k@L{fjpX({2KTJ1UC^X&U=N>-Wk^ec*)fuq4{O=$2oAUp~qyOh#K3_xrZ{pYA zu`gh)@;vx~a^hG0{=J6BAH&Z=ixV0r#Q!j5QuEa*z@m(C=xj=_FgJXo8FGC3PFqp6 zpj9cVPtQ+f#$%nUQi|BT$U?>vA@x- z6Q>vZq+kgx<0}RrY3qBMM&1rd*#)N5-OVA05{nQU{ogwQ=9SJ;s3}MyIfKM}EbxLi z=bz8ZtYF>_If1k(T+H-v9A-$X{s@XLC;8y?M*MATP3iu(0%kIAXMi z#f$78UU0?3EMmO}-`s0_Zn6K*-X6dC@og}R*6m~Y{y#VxJUgiE|AXg;kLUlre0Cbi zy*=+(iB}4wS3r`e=z1~G!PUmB|zNs9-s!#`WUvpM+GS91dsUaV= zPOXek5}HPKl4?<=k$gEz39Y0E$KyTlymzREa`Lf0u%*PxMPC+-^)9B}TnN0Ll6<{- zQynnZ6^xdwNgIqdq^W)lGG%cA0KKQ2q%tMA7$o|oAunk;ZSWx=$K<2qTIU=F-ky88 zF5Xp1Ksz=62{}0O1RlW&~sWZ60ccHnux zh?^>*DPBI)kYOdQH^EU7msBMJ7j-9Dl4RGIBAul|-14Ezmv%u4`!&yC8yO-72QPwt&<_R;DunDsf3&7$MX!qgnCxWE)#F_bq_9XFl`Jgb zfF&dKrfIYWYu$sWcNx^2>6^n3*2$ooM z9G|`mJnxtjDy~uUq}(Tad-9SrUpBP8JxD?dY@Za2Pqld~uE~4kgx-<+WX@TE7Bcp0 z&cYH`P0or*R?JWabs?vcq@XT~Km?Ckuc{+34~i8E=DLx%vc}>P$&np-!LEd6m@IU` zvY4|`;QiPC{eRwfWN+`MtYpKzJ?}g6u5x*lxo3rbywJ^Hcp2Pe*OU{=c^1a7GV!cP z7vdsnus)8NiBmG6SAw15j1>YQVFU+=%@3E2H}L9IPsk#hYwj(wd7&@Wu2rgT#Xydj zZ@4(4Y#50fq-ttJcG(q61xG~(Gn&q6k}OD=Fq#U5K@YMc)=^wy8t;Uu3;YkFG08?; zDB~i#WGMvZVRobrMK7O+Q)KRBUO*OIXakQ-kWJ8q5ZiEA?@EGGxg1CPwIS>iR)#?* z5S-GylIjyD7px?_oQId%ZUB_+lN@Odvy?+N@+s5OMr=!bJ$en&O)&%7t&CU(j1uQ< zo~2^>foNE^mdgyS&ilv}xdD#G`<3NCwfPF{&rMwm14LMaEkQ?%b@vw;D!8y4h zZ-1q;JYnRR(V`q&1Eaxgl_D>s+r&XB{&LaDdt^i^z>spFoonZCFlj z)e5G`PK_FfIy+)dc}3$DGU=M4GhQkosLG6FO6qTAEV1u~-&#F=XEYf>(ZRD9uSPhe zM$U6Zl>PxMG4kLu8}x>Ze19Za{j zZbDq3MjeEbd3F_INuK2?XU zkwt2EXmp91d?C0&$IJ3LfPJi?KIP=CW=saJI0Fp3d#^NwBW+BN%F7{oT zMM}kyzgwmOP?&`PN#YP@8%^3bWc2FU(X0A6WGqB67LAFm=s2o`R(7(9J|Q&B(vHfa z&#^CG`tAZ50TpAg0UykIt2Jn9R#G` zxRQibi^6htZIUl`9rn$6b0AjMM+`W`B0{UmHH*;ebsOp&NoGg#J|*pJd`t5dbFqY{ zStNyc!9p~8Ez}ZLI-WY);T|xf3!z$A_615N^KuTCe7x@5eU4mQ01W`Il~e^*f(cMZ zQ=UcJTGt>TZxM{SE>mxl_;(u%^WS4tI1>;*9DD9Of^v~FL%y4A&A=AMPk|F!rEXZE=`rY7QL5jK~xxASj)zOzS!(Y6N9~Ls^7IMQzNWw*f9Y) z$7^ZCCUGisc2n4;)f9|I=qra!pnMxLN~Bh_t!w)o7$M5H2?DXRWp9C#W3^-lf?0o? zGH|$9Vo6y+*0uq(h$&YNV31b&t2!qo^))-0l*?ocxP3K$QiOm!lOj%Apky=D-I0Pe zm`7bMH#?joFVT65NH~R(jkVdP7RX=B%?8OwXuxMb!1UXPqtc71;plW@5Rp(??-G=P`UWId zYMMph2E`+^eGE@I@}`)lcOFW^5~)3h(w?MUi6n_)4vDqn{$XHc%lrpwzz8!bjPpUiM%1E2P|RE|Ea@R5pY1v`wUG2t{tq zhQe$2+ovQoJPFIp^*+STl!!Zbe8B>;yfVVX9X?>c^`36wqFIkBnj&Na4IpT zq<1(k3bCA;2+OCOTw#+aHW>%Bfb=5s`bdq2AP$a8w$Sc`8O_BxCwc{xAU3yOF{;Bj zVMW2P>|rB7Mh-q=srG!$^T^<2l$#xDg7ZMT3^erxBP@l(lEEwdXbuX2XgZ-e7<}ch zFcx8FU(%2DwZ|>blNb#kSrM@!kc-6g_CCb<5H_+<*o0b(ao|!*T0}5|G%Q)c_X6U1=ZqP|s{wOw0rP4a>)=$N10RqB0D&;;$pM^XGzvWN?l-%X4@E_Q=wVt~e4bELrny9-NyQA?Gq86}(=DGf}9}I%N7a73K z5dEg`!Xc`>aHzQTSH5u&gvwLFvrceetm5laPZBKms8=p>HpDXsjh)_=A@m^Vw@TwR z_P=%P-foR8-~WT>gTs3K*Zx8OasS`TXGf#NaGK$WukQo>@((<4e?R{5eRq--Gg=Bo zODHv>z#A4OLYcavMU0O7rX>szlKTC4;CZHd2(YN`3qt7{XQlO=L?IFNMTc^NmWR^5 z#2EbJn=HI!g&@w`ap~4e@%V^kVvI&(_TuR1=x}iK>eb-j)vJSk7)~gq^jV+2d>$UX zn6Q_Fp6P4n{h<4P@Z9qn=uCv;N*P-wGJ0#ya}_;OUL#GoYC=$QNe-RKD3(Lw`}L^Z zwYuKU{og*k9dGA>ZXn*y9qu4N_x;{Opv!x`i$IS*WhLW7Z-;cnc#Cn6R@9n5mUNA%%8CiiWhh)${czz&%kjJYkN95wye>V1$`Jw4^ zYnEANgw;E=fdYnY4RUrJkh6T{REAN<)1`;R*ZXtTz+_r^FTn4y>&b~hAwi+Psi|1)c;#$8J=2cnZYrZWBe$9Akelb>B zPX!Qs_3Y>swD@j9=BWf^#8TX=GfIx+m$UaZ6RetA(nOpqh(KE~2J4Ce{N1UjuF;;gm-mkD+d=!RmGXby!l;$m0yQTDkcL!OqFbP-TFfo8Jjm4re zvw9PTVFTBoT?urN$BXDkEKOiC>}JH@C@fOkjKf^b-ztGWMZtKUA+khJB_E7%-6iZ7 z)u)7R>Aww5$|CCzfI#xpBL2!aB($57((PSu##R3$yGlX1kbv%+=2OBC6*x{ z{aR%o7!hXqLdXIBZh6nMSlqZ+)J@`XL5oFqJm-rs_!uM!;GP+H`of~F6LDIKnGv_n ziQ{p&)a+v$QE$VHbg=TU$T&CrSJ%SkT9u86ehaZs-9Y@TFcT&{&=qi*5JvJcSk6@} z2RI5S36pFdsXZa2cfqdV1#iKwpz+fz>VC+&KZx@~+JvQ|HlCM&%9bTgvj_+BLw3!s zSh4>VP;Yprg?k>2u4ozxfr>A?nS2w?1prw~#%i@%W&OkX5=O2b-5+cY731bYyLZ>Ulm&rnj zyCS}rmX7YkD2&6+X%LtD1Bwqx-}B)9w&L2ZLMq|)y%FXMZ!UV7Xo;=T2;trRW&dU0 zBV;Y%C5t)47ov{D$vaO?Fc-Xp+P=m$CD;Zc%Opi|N-XuWOJHF(q)(4Pkl*uX4gar+ z_xyRvlAIMhDDzER!7Kc~herqX_^$`gp7$U9zxVOkQE_fO$kQP&0PPv{97&+6frgwu zOI=N+xcP?n{984k;?Udw_IAkM;xx*xIXTDBv|=#5Jx_hBEsR2B(rX4FL9~ddZL{%D``-*+8E3akfjnkEG^`obsc`l9@3H=W~zxN z+HGCQPhq8S(7Bg}uk|u@`6SzUrJD#W&-;^hzS&Fz8b6D^XRNpY7vXggm&_*Iv@d}h z%XUa>gK}(k!wo*$%ys2c>VhMnCKMX}O<4>Jt}0bo7`jq3HilK&wL=^7oWE~c8~K|f*Z3bQkm)sH@{AAiHUn)PO)n+wbY zfUr{1>7P4mFMA}anl%^<&_At7r%LnLl?L}NT7U18`o!9`moIsj=J^JB;@S#U1VTWX z2Mv^4x_j4{Y?&SQKDbp)2fGfl8o7CI&8u3vPWSQQOwQR<61yNSDiZlV*w>QHvkNH< zT|8WoaP*eET#Gm>HK;y3P@tZ@hcl#2M#6a$7@j5fTw2Q-rBh_Kad(xV4T!~bG?bn(SYv?79KJ@}lPY1rPu4Kn z7C~$J`C8IZ)Gvf85D?!i2eWdTIXtCi?CR>tw)=V_Z*1=($r-Juugy1`tm*|*x7{SL z*{@H*6YLXT#;Nt!s)c4fwY7JFF#q#s8bt-;{PSnOvsN>uN#N@fU;h3%j@Bc>imdCe z&x|H#UlW?=Vs;$AHM?z}O|kr2=5yt2DQ(<6XuIW2W~@^0E^}6C$USR#9<0~G1~XS- z%kWBjDsry@f*c_vg@ep?5=*SbX^@FUed>#0r%^Nr)o+cYSx## zG9jc8M}WR`&Ft=l6j-W+|5MjXU&#f8zFY@6?e4oV3-^RKvin8V{EC+x3z=ljaP+ z=3a@FqRUoey`Wz%@(rh@8B=Y8?kpISUb-lfTlwf;vc+8}tOZmfz5+*Wt5Mb7Y8g~z zhZ_Gjrob)!-TnPr&%t^r@Wnr6F8!t|{QyaJH}e10uqeC>A(vJ@3|;a5`}vFKb^qV< z!SjPh|KELlU_ENv#1h|0u*BLcQ~NJ@yaARI^Oky9a#`;(PNN~Q63lutR?>);bm$R+ z51-dI`4NKF8>rHLSx>YP=7ywkDt0$H6!jtC(r~%UvzF4fOK2YdrO4(vACiwxF?rI{ zPgS^w_&!ol*Iedkf~90 z#wMa?wbqu5!Xu=)r#7aU&*5R0Tz%MJbv1yeZevamVnuJLv!mg#aZR6|q#8MQnxIXI zy6cpRLA7j$^)_D%N5ox0hKF!Io~Sx=Hs;w9E9Zl-k&s$M6S92R!G;T?CPg+m7wieD zrZSh4)yPEt$6CQWXMk_di})%|*o6(hw9v2`YAL>ZcY0DSbo%a%`J$Mo$NU%03JFx4 zCu~Tbo{4Xd)5X(z6okg_cRcRYgNEAd%05?AuD!{SqOS2W;kWgw->8Gn@cgK@90}y* zPQYkL6RCgLeED(COW@5@{1z~}>Pb(R%;I!N`uZu)jQ%<#N6()>d+rf3;dsVuskwsD z=;t(9oMjn2WkJZ7nRkrgEF0^Rr})BYYx61RqD!7K88!N;`a*@Z7GJ;w=)s@rw z(xLy~>HfE7OrTZqfBJ)3{I7$+h9_Ic>^PCIax}{b#7_n#o4@v7azHRZ6 zKdP4v_Ey;9A?on@Z(o5SMDH{9Yn~M)D~80EP$7NFZp1qK^@HQcaRgE(6=WSAFX~Z& zhXTC}r|)k&)FW(C@l!TwO_ccDnvpN{aVwdnAcPKv>CnouO67G|>wQz#a&%f_8{BZJ zZ!)a$I03kgv9!=`H@p0PTq$`&>TOqut-Ka#p|yfpUYA@OM!l3;yWk39U5cgnL@7rd zTHkuVj|*P@EiR`&U=>-gdV%?A^V@4nnf&Bz|MwppC%1Cn8_I3|otp^uo|Ulz4%OM_To zqjBwqShYAUF09AwUDEAN{u!mOpX|zUoOiFF%&o~=Wu~z^ z29rxy+$dh(BGSdJNvz!`G&KqS^t#8&u*Z1<`aZ6H>z_dm!BngDF43yE0$i_TOFmK+~=13fL9dX7un1XazQY@lr*(AzQ-sfyrkuv2WZZ? zlT<+3gea&JHY6@VDwBRL7aOusCAXR_FGB<)NYrjn`7BPdv}C`QHebM8^HY|JYklP~ z)<1$RhYb4tew|NSy3@n3;O70K*eJX?^GfcDirpwKED|9-70#(+HQr*onf&=yS^Ldz zyFWwMG^*90zjUs*&B#hdu7(;BX88hm$hWFNR0A8VRwZiXSLqa8!l$+X3avkGtd+pR zX-52WMvHJ7rx#YV7dTP#g~~hy!KLs~H!eHB(TZTwkUaVAg$|Mv4w$3MRP z(kuv%{+|5yzu*3SX)%(?~9Dpt(o7fTEcY2!Kue zCY9uda3N*7;@U7}G$P%U3_4ann%+&isSR>)ERVW(fn zpZ+B0;d>g(_ntt8w(0p$+jrj z*Q?aaucErWcssvogX4|aMV=q8ZvbHLkL~eA8s7=c4 z+H_nonjF1l)2{o++Nv=6wHylC7CefIl&$#{2M10=+opPT7mws+>UF(9&9SwrhfOv2 zidS)c!j((Vp57d~dKmZL8VpfxO&R`&RIdx`x&|LV{{f%d`F~L+u800ziT{0ecz9U% z{~jGZd-VU_$LDtbU;7NhC}eP(U-s(sKH$HNN9?P7h}C!Ue|+$-;}!oyfnK*h7hfIn z+MswFh*t+2eg}Y82UXif3$B>X-xGN9;kmIkbkhFvHrPoG+LL#a)fmdHzzU72U@Ne~ zmgepPD{Lsl{~@~Z9rVAoGXk#C|6aVP=l^^5;!*#*pU;Ene+vHBK=f@-WN!hfw?bx< z7N8!6dD0?%W7J%Gw_5jTytune^lUF)f;~ouQRe`1!~PMv^i!i*O9*NhFEj1sm<{P^?GZd3szk|{!l_& zE81D1?sal?<<`2w(CzIIJ1f>^1z{&6{5guL^P3@S4_4gmoj}8Zq`rmzg?)Ltp8iua<45SgJ&Z!QXwHJh0 z!7@Pj_Udv*1(aMv+^h)4_!Y!XOT@@l`D1c-lHYz?-JTWX(V>U_BO8NCIW(D4K*U8K zOdv8T?Cb*Fruu5L=Dex6Y6GOx1lhv4tv?%4rlTi1N9pi2@qJQ@S0)zEJ5KyKy`o8M z#RHYR?T}4t1gb+B>cz3rb&(~{AZqcH^cl0O;+~$s6K0)TyS0zk-#~JkX5KN`!)u|I z`((Gw{uj^E_R(}K?%84c^jKbtHY$2V3rAK2zuAr&@w<4Y)v8FS1ib6$5o>gIq|E0|6ar6kKt#b#R-iQ;(wSj zsrl*@U{S_6bT*||m>a&)3^_i1r>&@3(5e*Gr{||KHzXMgbg`Q!e-kIxP{rDe$q%xs6Vj7Ej=JWe1h zYfi&Udck<$$;`bRlk9QArz}Z`*xzW^iPMXHQm}-U@f8D*wDmnrBX5VK>;hBj?&c6g ziA9Kw{_mXt^Gata)D$FcONU!^_M(3plo;sGGr+Y9)0?(GF*dg<*2N95LF%;zjlkFSz1i7O~!gZ|*fdYwZ8Cx5saOd>hQ7 z+xD@1{~sI;o*h*8|KP>5{^R+7FQ1)8a&ON&Ryje{`9?Wr^H*PgRLPJ(?ZOj!!trrHT!#3faUz(w6jmL%CV zrbuV05Vw5jb;;keIZ2r_UI~yyRmyI$(58@;RVaC5CMN-sjc4kZ*6xuC7Cl-qT(M)ad1*bk!k4q0?8X5LWLkGJY6gE9 zbv*BbwQ$8olyawFa!*G_;L{g?>n-$_fuA~;ohG29eG!| zJj&d&LO)*UW-z=AZnA623FSNsV_2DZR-_AY5j9vJ$IQejnb0f2PI1NxfsinQ1H|Ts z%f=gcb*d+1k3OKmp*%JxZ)G>2Kr zAshLWX=x+2CB7cL2I;1l0qs^sECWV~^ES^?vHUdCKI#!mHven#2Azw?LSl`;Tf-u&ss$2TxU>1tdqJBJC5Y3aTh5@RU^HQ z(hI1VpFpkRm=Y0YNEH_6BBUf^r4X6&M5}AKavXvX)L+$PVzzN0 zz)HZrBmPp}3h#!Tst(8vdBb>E#PHyp+>p1w(pjD`a^u~QuJ{o7|8Dg!C@|6*$B5hz zIJR}JSEaL#83Z^$?UqI4MygMs$%!^BC%0+^(`2Vc4Md$Cv8TME@d}xAP0<-Il@L^A zMlvP!w=$O4cf)V3p1w1hjG*Y?*^5^r98x3axgtvc0G1ee@R<#ILq@(olC1t&W*u6} z>Nj%A)?>9Bt2Kgl#O4=!bqn{A-Z;s|y%`naqW2~X=kTydP~4+=9L%DfD!9;fe_486 zvGi~?y;5DZ?<*AcvJqVHhUgf{iZ7q4!`8?mwL3JrL`}XB+@RxS`5eGL)=-~v@>Vk@ zgIAmZiVt(vv7!*l(HsOp!P1C<$5(p0K)arhQ46$c1eaG<5}IV`1zhOhWCO)&Uuat) z6AEI?EqxT%AXLQVBu^paznm;>1UQi^6%8Sxubyo=xgn$9zDS-mh_m->#gJ05QGm8A z?(!kc;aJEf<_>BWmbH=OJ487$d;e6|t{tFIP-BGd^^_IN+KD(9`Fxzjd|J1qbj6yC zCvm7&Ngz}!>=uOnJSk&wm>9E=h1E(*P*ULVzT32(yhQ?He+B_3Y?X zeH=0tq8N+D#8z}1RYEH}*+icZ8fIxnWzpx@moM@;9lQpxbDiU1@a0RuuR@jXHGlxB z&G!isD(;VrEE*ZAB_C17_~RUbEIt@q62k_s!&l0QYcEjO;Lqu1Thk_+O~5@xqTH2nWsWXaV2&^P|8uj zS--UZdZuzg$oCoO!1qeWvM{oOrgsx{(G=Tmzpo$YWBn9+q$EiC&2 zC6jqMhf6+QckVt%E-ruufY(Z@0xQ7;sG}**qHV2f5RkVB#$1=Fw@LiFjfMH|u_~Mi zh#!tUcOF5x$eAJEO*UwT5WX~<)GX>T8eL=!EhoV!A*7D3mH8Rf>?@scuL;NkawzCxE+ocw(=Ti0~H7^XcdZydtQqc zDsOo%t+^UwR9lwBP^AJ>jbCsY6(VJ@(KZ@&P}2t_(L@_<}76bs^R7+LfshRyORIfSlvCG-8uD6*{{q?9yrqMkDl0 z`=QG=mW5plqfw73D&UZB{vyXdq{xFc@e zluhbo|7E{I%I)S733N|oGbm5nM4EXfg-chOD4=lp% zWiNtDN4SKlkLH}sgrXW|DK85Oz7}|0#>p-o`7o;v&&!YCOqs=L#CseX=>W5(4|YLk ziGC%{1M+=V;9`*X5f^3qRnSY4pzaT+5@Skwhx4Kk%c+U5e9FlcHi=@BaX<@5FEX!> z)MyCe;J9Q9?M|4{T%2>FS3n73bNdyeI*b!m6b#EAHUebi;3Jl5&(}PU3{FP5*`X#l z546ibQ(rK`QaCIbyuy#>pb&_r6Pkm;R}Kqf5q9<^{a9an-10n$(EySa5i0_@NIY-v zL!1v`BMXI1sI@3Jn~LkVLbbZNWQ!YAhrdI8I0X0-KEb|@lPIJ`1T#p(k`;U}Af9*5 zm_fW6F!vTPucomMP6ayf0XYB=2(z9ny(su2fZrcKEAapD*1vZGaGiCdJvKk9-v7LK zKB&L{IT$>?|Gt+Ggc_02?mR+5TGAxDP^hsu>rFwt4v7!JM4UwcZgZsqdp!N_H5mj4 zM?t^aC*P7m@N6Ld`P5l8<3j>*^{ijzk`zA+c?f>Xp#mtm%UR(+ux)EScQG2AyP`Bv zcQJNH<0(WfUfMLC|j#$Ls8W>)gH78e6{q2hRtG_4u#-!{?9jzwYI;qfufw z%W%Zk_kn)-2OhY;AOHBiJIRU}Erp^blp0as4GR;YOkL3;M#p{A5(WrK{hmDVJkvb{ zSXB1~p>&P2(t1v!kcj%CLpedqLup@P4F2&=7GAPK5a;cLsYH;xC)j>ZDCzMk9tWRG)5073<*vmoB^tJPT(0xC6?s*M#Cc<&0j4cxxy*1~# ziXJJiktSRdht6aa%OUaoderV(U2o_9Zy(-{xAQK)oZ0mHTiIlB(XS-x^A!>Hrw(nI3w zf9tlucw~uH8`})6s~1+rCgfE(cktrbD_fp5@TAa+B#k$X+IEaF&GgtTiK2&+f#(Dl zc_oG<26MFGw9W}b?p+jg%=Oz&h>pS4d<=ost-x({OPfM&+rlL-1KdJqUmtW^4UqT6 z^Ry&!tzQfCsx0v}->pq(-!}68kX8ifdX#pe-1Kb;SVw?o?FQXjN=h zRtLWbnu`^MRm7q#>NMomgA+jmY>j?bQxOpXgRhlcAX87DQ{s@>A#XBPam=a5xKIcT z+xq#rYV3cyl*Q$(=^={aLAozR4?d%Pn#j_IuclI*CSVw!R0tYd=^7lKJ$7scqI5khM9+E!@0$stOTMefIbz3auOC%eHOXwr&2W zW_M#^V@`HAHYQ@fjrTN9Ga~cOe17+J(VS4FfZ%{71xd2SF6JQ2cj&L&2DIp_&AwWo zx$qlr;SatM+VuZX#Ga8BFquFq(Fg*0BkV9$t6T*@m$G**^z+7wsjb;YW(k}40vZbacjIC2 zr{!RyMwop2z76>h%#PkA(>SPV5p;X~T@R?gL99=?O&G%2hfx&YKg=#6OM}rBve7%p zK(YpjH%TDkBu#l=Bduk?o-Yw67`=IGhykpC{{7}Lk(`FcOL_yZ-bMZ187K_=F_M9W z;-4d5-mc17*zotxqc8$q;Jo5UrIvYwB{P52lqhYx_jm!Su%#T2&plnZ_bnXJ^;UAz zj4V|cGIldcJC-no2x}}6QFADt`TGdK2+-#gA2}3Erjo9-Jn4;tp!Z+MYWpa74 z>dkW~+(d@cw+dQTkd4=W+e(wI1>@Yiw>29u?UzHO0P-uTyY&;HV1Azae&f@mK>JoE z$rwn0EB+IfyNXRq!hyr>#$$}GX!120^!GNv6#)j0>xVDxLqIYk&8xIFRZEV!*`ySf zaupl+1Bh|iAe=Nzo0(21etl>oNocy|ez>ca+P#uGC64oT(VkV8j1J$JtL_;Y@WjLS zzid0Ug}=iy1vi+~biJbPI)kJQQ0!OIyTkRk^XqvynOpXOGi**eZ%OUDkfl(;A%03| z13x9`Drh0qi&4=A$*xs#Tt_8bm}V*6sp*j-g5kgeixRH%hZE=o;V| zjksX2P#I!M-+o+~EPa-xx139!EV1*vhCqB$;rJrIO%Ez3lN4ra#*0xgP$;yEiI+!~ zk$KF@ueSGJ{x*3kl)o6+xve(sNj33okc?1qXSQx0)}T_jN7m z>xx|7`S{0I=X!M{y|2VLPI*Qr5e}rmbnHsHswsOVkwG zO$}=Oueyx5@EK+xD|qSTp4A|484~!XF>Ij?VqByZ{!^ZO(%0!O|pW1ZF z{P0#_v(~opFxAsFlUZ`eW<$tHpC_#~gA?lTrtS4*b!J0Jf>WJ_E*;JkM=KISkK!bG zM@;+l_4$a@$`a5%sZS@SeUGnudGTYusZSdfpBxcwZ{_?3|F!{uXk=uLTFn%I*2vh^A(pV!I`Uruh~ zw5A3?4wti6QVUm>_4j7^oxVkn4t*Q5#q_P!b+S5TyEW@F%=;11j=g3E;}YBHbO7U7 zQ6y1Ya_l#$)@E?OrU2_cwf$l7AVs^ydD~)=tk8PGu^v-lx*>yj}S` ze4McxGe8eEs=oul$zo|>7WJklx5r)g1;ERXgTwa@xIO^6P~0F(d6f$wUDi!s_ZvHa zt<6enS)53i@Jw~Uf(=jBU(T#c5NrG=9E;%n+Sxo7i_o|&ko-L1SrE;@P7&v^zceA$~N;B=j|OamYCyP-p@Pu5+Bfh%p{JC(r!4E1`j0jad(#0 zASG+>v+I=$Tbbjc^S5{o8fJ!el9b_|MH+Sd=!f?jW_3q2oNEPsoa)ui0~U{Y>_r`w zTv?K`FIMcK_H0|cLXTcgIHLC-_XX8VQfism5P2(_nvGHL2&Z_P7^l49;5!b_2f!;T zfOuT6odOW3J%zJzVMC*>-XBSpM%xxZZKM`;N|b+>n{ZdQylHX}T|+Z*1wGt}ETuOv z&ZMYw;s-?sEfQtK9Sy*-X=AWb){nL6%PX)Zw`Em|k=^qzLmzX3czfv%syX!2S-%zq zhb$>b-)(g)BwFS#xYE$%de~M5jXaRH=Q5;d+Ohh<c5YW2;iwS2)d<`5=#5(B$+vV)H{GC&g_iQ&t zD-oVwk@qltTp{H2{Qg{Xij9LBVK$YXLX8sRFB#jXnb@U-5$c}oH2Q6}rk=_(gf6F* zDbI}e`HaYp%%E_97~do&j#veNq>Sto0^_#Xp; z>6x9(yjgADKc9O8p-K=;{gN&FIhzs{)mh7_a9;r1m!2m>;Hn6~0{)}bOV7Xn`q%$6 z1Y`Ff1oK}7!JPjG!Tblo{MSG*TJL^}6xG511Hp^{;NWuj-~Db5oVWosJ|PAuOs!M& zUM`;aw>!hc&nkR_=iG1ne6K5a5rBMw8m-*O8aNLdg-+PPia^*Nklu>Ga+^XhHJOte zD4w_5O?7N83qKF+%YWckV;y6bzmHV)gNK!CQPmBko!tCU*--WY2wSgo?>p_7KtFDT zUU%^UfQ!d-?)UQGf$$~Y2fzyJz6!sjC3sRDwVnrF9AJ>w{AU-HVRhpQlh^h*d=jqzPI`@@NjNnU|TZwX0FLudQS8r;nVYGMQ8k!BQR}x$r z2l0zDy6B>4qz7v-k>JZAPzBWm$f<mjbp%PHb=Z%MDI5W(2Sd&a=+*G`s}UP-#Gd>}u%Dlm*6E2d5(Jqx1wX8%3h5 zlljTq$ltD7maa4e@o+s)`Rf@=lY|o~!9=tl`#^~j`={w9P7*x<{K?C0P%%f}uyng3KK zHrgAD-ez8vBKAa@X;wu9N%eNb;MIXE^M`wkQKb{D^q$mInXZDeJpE24HIR%`n#zr7 z(q2~O!{dklUXuZUv`I)WCy#A)7$8^n;rjtzeWKdWyFC)!=urTb$K9Uu z$TLy-Pw}s@n)w_Ha~Wa6yI*lvS74u9FV6vtU60QJj%452sCU6FFMgf4_iCDhYgvhI zcH55hqdx&KG&EfVF&I|{cSIkd=-PJ-k?J>n>@QfkF3*6qjRSW6ZdlVf+7I<^0DKJi z3|#q#-|ha}-v1f|WBDHh^B)BBUmn2({|CYR2f_SL2&VbJDuUtq|0{xF{SSiq4}$p* zg82`E`458me~n;H_L4*&f>67V|6W#N-qt;9Jbj-GJ-mP93i{n*{P&zr1WOUIjt596 z3(3Swq5R?fN_Mb2E)qUaWCt{$)~RIZ%e%*p6=MV(9I!9jTyJUcz6Sr-6D6~yD0ryB zr3T+*=|7(|hlH=Rq0o7UbY{VT-sY;O%^`F9>xg-gt)W%i=n{mczoaZPPq|RHE^u1y|kBH2KdBSG*&vY;?;RC*p z-*%?HTc)?8S61)8_}%$9Do6Zqad7Z>uK^#gm*z9#y6@$lAlz(#1x!4=Oe?BGV*050fh_l_5&v&YYt!s*>lFrxSARa`ZO9izMw4g5b6VlunB$8tbZ zh2!nH_j_zael|6Io$xL=#DU?q3paTjCWL~Rm^$t#?)N*y4E!Pdy}Qa$nD*!U%&;#L zKZ}p;6w^dht!F5kLT_!A;B+;JV@gfdIK#Pn`(xbe&u=PP20RVJrC~PVi}T>*Iw%j+ zY1~ft=kXLXSRTqoBFO{U!e(3}8IScYs#a8DBB_)K^?7bB&^A?u>e**bjM@-txR9$} zo%{iQ`XrV$c6C8_FEJRdgFZ7jduCoX>4MXI<6X`^QQdID@ID+Hjw+}AI4^WCT&J&Q z&~{`;d6NG|{Wwb(v9b7{wdrot02L+a40B~#TAJVG2r`@{gjRjo zSPv|qzc&@9Cm4wvXoTV{cr5g<6#rsYlzW3&^^7yk>XUTpcOHJUp4S*nA12nclAQ5K z5@+3N7?cqjUV%0>=OkV1v4>&dyH9)O1kK}@Fd)gJ5 z{iRj7v!pB{(z|PAf^hnmrGn#h&Pay>JkadxTaERROK0NXVdWXN?;dw8 zJqI!deF?QuIbd25y#re*Mf^O|_PXU-4m}68|4VG1INBa7h)WLWHNDh{p^zG2v2V@T{G(|{e zgkK~Hz2T1&2P)?LWl0k`mK@XhD+QWmHMp%agl(t@PONSuV$^a|i4~!T@Bsp9_NS@? zWdd8OazGieC=s(HoV6(}$z8wdmp)KaB4hh!ZODG5{GIPSvTRAxy}c%&MkCv!!XX1^ zD6UgiHdVFjXRzMIYNYn3yLU(Y3M+z=3E7Hz#SRvY$aF}n_~}``Eup}eVHQb&EI$lH ztnLW#fx8-NDclS_ z6@yHNg?bg2AjJJNdXSvGJG3_*Xpz|Hk?oQ^bmn!ZF(=-P;P`NYup;ABh0H4F&G}_M;&Y{W*6Dr)9uqVPFFdT=&TrLWn4N=ku&xWWIWCZ%RvnrZP z^ScjcvKHpGPz-diHbaL^>K~&(%4@$Zz%)KGS!-n7&tg)D?|%-&$q0dJ5E?`osGm9q zzcOi1IH(PxFU-$nV+1?2a>DLA`U%_-o_XPI?x<>Ss7R1@cC+_;?@fu)Bh%KYU=en9 z^j!8_dIeRIb{aE=z6veIb9kJTK^+IADr~H@7K3gG7^nY}kP9#Jb%mmt40$|5guM6q zuDk%SGUoCB#5p$vK$vtFQ@aNPUT#nGVgT;0tA0@cPx@S3)RW;r9^8B2*&M-{c+hez z^U<@0#U(L_cpV1A_> zLBE0mxW>E5Bce1CbS7$SewF<`NHpGO!=qXtPsJ8JKmNZP2BXBS;$2mVB)s4Ab&^Hq zo2*4^oVJ;{j8oWhN0NCY#Q)NR3mQ0JyHjJLTgwyz<`*t~w2XWer6CzuzBH`w`e7XU_IQ$`_hCIB>d)#k(rCY=V4kntDq_-(x$s@nOmoYxY&+ zk=JMltQE2*velSvuh<7_Q@vSO6FbSM0o(R=&9s_2_YXP_5NrPIt=+hxy1h!-c>ip2 zMjd12d^PHyl22vwgUR_9CZ~7`3Pt@itg-80?>EB^wd_0Y%Fpa`TSgL7#v0*xN8{I~ zv31sT0lv~CRW7t&k$v8KkRS=DtPMfezg(oKHN!pDc7;!eLO6v{tz8$+THyl4L}`qm ze0#boGx%8_RC&)j-$NjFZhzuG2keM;($mTlj2o4x=Ow_v!K7ABC0rPnNZ{Y0wA9kF z^DBKHExul{x0fo}k+-z<$$kYTC&&RdHN4h{6_+ae^#6Wa$a1MH&s4Lty3M_N!~e*1q&rS#qtqq%Dn3d0FW$&AYV6!kalx z7bmQOIVi@DzVvmcSm)?S7@Y3!MFbVL93E*7YLp2+?fC;gUGioT+AtwaXZGG6j%5`> ztSozM7H?ILA!umI?L0@gywqCcpRHNg4Jj^@b#FCquJ6xlE>mO^HAHt*fZ7Pq6c^?{ zM)hHVC?DUl?&l~&f%w*k%r!%5F3hYX#1!}vD*OL^HRVi8UYOjYWHnM#lW*QyLH026 zLYJI2=k>AaUNcsqPiwy-^PlqdfzHv-ePxoX>QSX-xhSjZf(d%d^uLsT`^@$)fd;rz&$S+F{D2Hj}nExE9=?wPP|$ z_H;{ckP^Pu7xEI8M|N*D-T@`U%-T z|80P0rdovk|Fqi}QvB=8Qjs9QnAJWg6a_Kgf+5@?x<$V;7Rp7N? zu$|?}h~MKX^%FtFt>&0Vu7o`vuJ6+WZcF+yg_b;4&6ilYT^DxvGc%HARh51vF*v*b zs$z<g>?&sv67?NzfEsX# zJ)N+Q#!G|Kh?kw2%$uHAIPm`@!TirjFk&J9MU!9v{9ZTPI4#jM5dfbzN_(?to(>o9RT3qnarP5HmsaZt){ zdso-ZJ^KK*Ny!B2hTnJ3{#QE z1*u+BBH{(yZGv;bolZ1*7fU?swPNdtbmPFeAK>?Iqvx>jm-I=P_*nH=?chsVFXF|v zIe1((IdLKr)rZP~iy=@{cZu0Z`{(m!rL7S9l<~~xiWos`a}cN!yu`i9k_Dg^)>P8kj;-pHSSwooTGJg?eXBVlY5l4<8awFf z3$ZRF&6YhZ{SFN2ugE{aJdMeoJ&{f-pD5@J__xTG>gf8qJKCN$S2Tvmg#E{@GDt~% zQ6}L07M;$oWYYPz^ExoJY`!f^PHGGV!KI8(MT)R1@Q$@L>I5Z4_*Ro)FP7zNUFE2GfEFf}ZsgZ#bjDc_ zZllxb3g|p&7Hd}4=nOxTo;t4DyScT5mv|+d8E6uC07~H_ENeRRJKlx)Bb*>aDCC2! z*6XnC95&0}-{iE|kKNjM2YfZ^R0p<_t$cREDpd|G>k#uN(f$g%)ZmEl*7bR+7R>qL z%I`6=IzcI%e%|%1K@nw1E_0*uz?(U6xzu;{MrY8<98pTp1q*JqUU^pOvLuLIR_N*B z(mpT^M?c{ZF_yCqxNF|6i zHh(>Oy>AQB86^HrdD>8{K@_tj-!-i=AoCddj)!babD(W!{$5Bsd{^9T+cxK;#i7NX zz^p_l*Z-z(5*-laujdD(FYj^VDRq+nHE{9dHNWShTdi%F0PjwDF(0q5x~PrLKvlSx zBJc+-fjbzEiw_dEOKD3fZ%!BYu9pJ4n7t z4X~|PCp?601lfJoVBegQ&$wCo`_8^D4a6@;%$xGBU|Yg9DD2eL$zzpVKA!F3XqCTQ zlQGH`U~?749>tbKj23Ov$;BlS!!B&OY?uCCO+yvyP=yF|g+~(w8Sh#{4fV>7XwJM} z;?o_gSiLX}3^B0Uf@DA1v{D_K0%9iV}idoia;SQ_(FE*>$KvzbuKVR)U-{Gld?R+QE_<$VLo`MOGW1Yg-YGZzv;;3Ntizr- z-uSy8?8H$3Cyt(7FHNsgI=Wg2JE0?15Lmtta5cCJTP-cS8AqSIobtZD*4CF?XMCN} zH~(DLiPwy~;E%dmQB}(Wyd0>t=eKk=p|yVB`5|w=R)6l0p#rVk`N`+9T>yC3zWM<^ z89QcsfT$0>iNc$2iDzE-^Q!zc>+nLnVvl-~uVG!MiO?>HlD4Dl6`$j@7O|nG{^bpu zvo16AR#&Ju+tm6P*WC55DwRxM)PL;uJ{|rin&jy3;x5`1S_e0VE@F+9cbZPvTbpLI z4e8z2$92nXCAQakv!AvgHvumFBlr>KT4GwqV%3FP$($e1N-gwmbxw{p%RHA91A5}j zPI`y+arrW0mzQAe&<53cM$EhQ@+#yY8$y=L2uaFTz;^kc(I@QabV5|T^qIU$ zN8;D}#rt;_Ni44SY0n{s`naMBC76V^vnnYOvEd2Z?Z#zAWX1y9fhB5OHq|I;SjS5( zVjV5*Ox1>wl z29x`o?!O48T3TLb7&8FSA)9rV5%Zvv4BDHzDxvStgtPlKYwlI2h3B<4o)w?P*Dz8H zjl|Dm-_@`k(X{11V*Mz)TS2j_y(;|SDukBFMC`XSJ84t)K;{pa#ZvtdBqeD48`G|9 zX>YIhAD2Cu4}JpJ9)#Xhx7VQ_jhZm6Apq>_>y~}LT)yuFz**wwyR)P$OmlKrk`3t} z-4x7=Dkk`6^_jYgr^lzc3mdxzon6Ea0$yJ*7Z$50)>|+V-l5bkldPS*>4ztU;2UQl zA=j#m;l4Y>ExK_FOnD3w>VJYQ&>orsTMkmbTV{YlAhoLSZjtP7nV9D49tzR>H@FX} z&REo9>ta4D1em~>z1gRL>u=l4Z~iYsz#B==PQdrhVD*k0 zSZ&ecPu&vra>d3M1Z^aLac(dkIggzBjWt==LlbE)5hcoRwgWYF?CvF4ZVtYBO=W3w z+3@p&1*39zYbzC`@F_=3Xhe(i%?jjcRKWMyBJ{R>eq$T1WNZJL7j|%0Ux4!-^9Yh zg3xjQMFYA5TkzM{5M|`PwnrZe9r&#t=nl+q&mqWbjR!*}+@s%Q?l~2%GH$d-DJ33< zu@^juOST7mAY;!X9{7+pKcua+tZ`8lVIYgM?~FS5C@8{_qgpd z6KzLGE@tLm0JYPn*vVU4Vk^CH z?1PyLfi=axUla?R6v+w<+y)u?Lj_e$9`0&UDy=!G=?=LH>>wJbRuY)`B4mc z>N?=tubWk?&X_}C@tC9H;8@}Bj2viAHb#L@NTzseU#wPC&R%U@XRQUqecRch`NxDu zbdMPp%-8shtW|11AX8}feIr~3$s?L{UbK9y3d@7MoAeG#;5Q>4b!AHy!Tjm`PQd!s}*@iAGytV8pcHW#& z2e@eG3<*vlblBeZf{qmnOnY$Vg1<|}(5hP5j{#tOj zMf>Wnrmf&t1KWS4`mBupIzw3w2ig4;CQ!1B$eiX3r3u$rdWNFX`2rh01_`2imFd-d zjP*2Gf2pIgRII7tL)!!j|A$e+c%VXb5|-8iNmJ5=Z4qz>LiAH z6FqRw6X07?A%eypQh2>zHPH+oZ}VShxyzwqmAzCSMqoBzi&Oi4BBD_bq{bLRvcug(+h+sJk1iJK zm>kB*KBWqkcIq++NCNgi>pten-@tW|&!MzN=MC)$CpMv%bUGF#hJMB9+971cvuT?f z%sadEH4i3Bu!=qXa>_~Jkwl}B0hMPR&?j%JdQsHY$Gj220Q!d5!puixALmF-B?=zo z>7QCO*i|v%5I&R}$0f+$*5Zv#*&J~>Vmj1kN)+wVlEC+eD9tzE^on)3sLcE ziz{{67Ob2Fl0aJL#@&En79o<1g6TRl<4IR8*}2uDAQ(XUmSHcVI@aJ!%A)Vol})U8%TY!f-L$T1Ws5$Nk!`d0 zN`=CKx>D(tG74;_QD4f=LFc>01QPnM9tr7FDk6vA9)MG}_tOoeYTi#r=aDDtv z_{kN1F~H3VAOIXa<)Bp`A47YidAVXwV+0rphro@R=-GdjeD}~9KPGT*`&DSmz0l-LglWrf`x4ARXhyfN8poPl2yJ`{`0zRdxQ} zQb7+Rr^^ZE1!lrxRNzk81Pd{T)j7T@ZU){G+LE^iBLEuJRiSPJT&bvp9@RnAobLT~ zKHuZ*mw~5w8(2l9WW|9t>r;+|bNZSH$di9-+R&U3DB=)JP4ertuL<`;KE(g+2 zV+HytSyIJTS;t=;T8Vw`v7HWdBs_0Z&t1ieHPTc!6_>k&HQ1urs-$_w`*@8fIwQd9 zt#XM;?Hh^pZ-(vkr^r%%uKj^-D{Uqe#%pA9yk$)%*R)9z$%T_`Z7^Sk}J)hwrxnK~ApOf4Btvavhwf4ZNdk1fY>Q2vH0MvW(a z3QDbjS^`}G9uUI=t35V{RnFtcabQ`(}QuEY&apKxeNZpI7E@^KWtR9Sc*9ApudsM!Q3iwSPJU8M#&oMVgo~TxqI7 zExMyML;Due=Exp0$|kU0L{n@7apUGM+q})~`UvPt%#g6+B=JdLC9NmP898)SG&;+oRgLeduHMq7TkZH}msRTtDvaJ; zPVy@)vVJ--*7m>tyGgUj!tg8iY>1bqi9~$rAwZ6_^wWXG+}O;8x97ZlK`K|RK@}5_ zFNL#F>(;0=dNHm7Nk`#yO>@V{Rc6*P<7$<+X-kXPdtGErex%VHry?%lh)s1O7K=@D zq{>saI(h}o2`6Xbl-HkuFA_8|u?3ondxmt*mZ8MB5@kuDq;iO3O^cvYyo#p#R$Gpn zl8WD##uU3C0gK{X4?9YReHk!=j7?e%{G2`Mz#PXj54I+^bw(-~_Y$fl2`St)z%=vZ zYU!oUr*H|~AyQ9?;tmsss=;ce=>Yk@---fbC71Rfv*YBce2X-A?P@9Il16?>k(R=3 z;~bYiCQj6pQo|HU#gA!Z8cN^it}hFY_O8tk-)}-m0L@S3PdKYX&I1xmFAL*@XnTY3 z`P$XgYNCNu2rP0OKMJx=lJjy)Gc9Hx$2w5Frui?^X7~wvO4FwmB@%uV&M)%Q9`afm zBftljTygXPQZ+6#cywvtug7*7Lh^{!(WDPNUyX{{Mk|VWFT@Oxy?^O*;s83BALz*o zeb3T2Ao&9TbGy1muwt}gr}H_pKtA=k(;<)VGX{8&^o_F&s^9X63hvwu&%&yyQ-?DZ z&=s$Q7!%>j{M#{=EO~jiyK!QTad`#N@lE#;0vjkU{51oSoh!v^7^gPlA+7bNXZaD~ zGKKgcZ*gl{65#K{{joM;ZNG(|5V1b8iCd)*Cf1LXWAZ$+rH^sxsd!}1EYv|M!XeTG z8JeNtXA>0&o@5I))7A%62VMiu1H8&vs9QHW2zf)O?CzdKOw0< zn=u|=blaXq{Cw|BP<@4U?(IE20QXkF)5<_sP0#iQ>;TkB7XSLuOis{;Jo)~by+ZVo zAM2wYY_7ZR1UK@QL-;Tnl!@JxvT40|0)|FWh3Rifbc55Ndi5Cfnw81*t=HuXKhLv) z3$L#k5`XtE?oonYFD1#s`lIqxk-cFxCWMQl7tDGy?DPvLqrFzjZ9PNQxuL{ZJW8=ixn&pXZZ3V58xOHVc-+p<9_MacJ-X&YYer&g&sO`nqa z=)kf~$?LW;j7AnO=hIbbi17+?ZT~)AP7gI79D=*$1^rnyoftP`JE(uZt_0_kaVOUh z4CdTr&8XN{_ijy2Pvuj%QX$chw~qZhe;CsXFXT4b^uxL=rbotyU4>kKn=BiF)^103 zR9&4F)3r>@lU6&n2KV`Tm27whbjIeGQA)0SyLjtHlXV+x5S^%%MV!kP6+OuS_toHE z`N8ty$*QL(2#d)aZf`2GB({*(U{R?zG*(_kTb2bM1eQZ76}wZUyk@JOTS#E6vJX$x z={sTOC;q6Kw{>EmipFvjunRF0mFhl7cRJ;KSNj#c5h;aHHSO9)8KlybfRiwef^W^8 z(D2C`4?Lh21bIhgI0ph= znB0*i!|Nq~CS5$EE2YRhnXa#G4jN&D?Q6^R0ccMfTIyP(JiqXtO!WjMbHai8ux_7P z=I+UCpo$&rk^b6dEw`M9I{dYnWPy{;_PYxE{YTY!s#U$nJh63cBWA@MRdJHEY4AQV z9gK6XfE(%H$%yHnHX!#AizfQpSYvqFCxk7Ge;V=wG1~qqHV~A_Iye*00nN_w)##kw zh)r|ED%%_3yiciBJ$4A)ISJbuM|9$K9TlgRtB9+Z3CDK^)(Onlk}WwtZ+3-J@H6g& ztRRAzes(QK7ifCdg*|#V^FFl|`=!cJ~0qb6Trr|!XjsNC4{GZ~Sw&N6rOSv%y3N=gMPN96l z2}1SGDr0!ayKGnD?Rtfmga6^p-gj9W-r|JiK2zqdy7RMGE->8Or^^aLMpI$nN@JU1 zG2}UI8<_w0*Cpdcj8VuS7K?h#bV_{PaZtp8D_Z650%FHHY)Bt*?)sI=jjRFYbM?w= zD?}KiN%O3;0B#kY2cA{9w1_WKtWysCr$ZTZ{VzDO3Dr%B@?=NTj% z`81y^@3{^;ab_jYWYHhg?`ZYa?TW>4JoXjrrVT{BNnyayx9H?w(w;12#jWHK=uZM5BFS=tzrAmlee-AL;An@@0E zJBo#XtdTL*tgbUXDqR`J>Cl`}s&E74rhvHUS!N5Mow`(+O~%LStHvTUxQZ z>&dEP5!JUzOmZID`x2$aN9aq9oqGlh2J!drfh3s>P)GhIk=ksWOGlu?gQ}X&Gn}ZF1?i}~4wlhFi9I@21AYU%C=xZ{m z^(_(W(RRsyO!($|BHPeXlQ+Fjc!sA5OH8)j14TtjsW$Xn8Zt{EPJHdo zIk8USkd@aJSAsHHo~Vfh4{NEUAW^8ZO}YPS&WmGKza3InRc1aE>z+cp!N5o8Jw z*1ZHjXiU^^IOthZ#997cHzyUPesjA+D(~rHO$$_5-gQ>VsEMsN>QVgmu+2W8|GK(! z#Y#obfjSUybI_mm9$lWO(oFV6;?t~~eApDjq7G@7v_enkmyFzYs9?rtn zc7&FvWAv)LvhPY^}(S=KTHcBQG@=J$spH+#rJyHzL)A%F7_hsk>c75Z7oAy`OJ z{HEgSNNlx76SH5aQqRID8nLR)u`B^oNR+NoU30k;+?w`vo z&^0qC*NmUL5=&I2WH487o{CvwR|>q(%eFs*bD~aSt>K%%2S~o5JuL~fu&XEk>Zg=6 z)3dYdy!yGvOZo}|J=+L!N_Pw7!5y*dJMvII`&sJ~X5%Lsu7O+*6IjcTLcQzgu#H+C zp6u$@a}O&)g96=H4E*B6Bi9A=r4kN(<=6!#?pI|-IKd`S$K{d*Y! zg)M_eV!fP2f>%fWpihUWn}ieQB%ADl!HG3e!D!m4Cugcx?<&Eg{fzH)fLaT@YEN@5 z*OZs}xn(flHd_yiSn3`}(=|z^R0B8MeAu)=XXT_E$W^1=WKPc*U-@Fuqt8Pbqct*h zP2X-sXOuUXDQM72HJeb&lSV6dB(qJ{MYR{+Wi^sIT+txRi4wagcJ#{jzZM_1Agg#S zBYjAlM!e%n8ylz8$`|Q{U0{Eore{1*j)ck{9=4ywwop`ncJMCdJ{~yMe?yWt*xET6 zH!^-?-O(&mEZqI9M&BdM(}ZtyYF&S8BsD zwrFv?oj#wZ?`s?8phm{na@8W$>Q-pP+PHXcaB4qj-gSTr_>JL7ftRmz^2n+^|8c^z zRTPwvjp_ZUw=!ud7O9nq+-}26Xai4cy$8it!g!(yuakji*`(l_=V%Qur~l$-xTAajnZyqfcL|*#NsbI zzt)$>$giIREq#8?twHDw$y%We>7p++P`kg!|00TXRkW8oq87{tvw$16wNZHX!SFqz z40a!esrZK#yg+hk(3g=bg5G-PIhIN?d6##D`-|o*6U_?$Rn54g`Q=V9DgDfEKxp!w zw@zDI!u+vO6;@)b8F_Qbqa&7}(~UQHNm7g5pbHSgADaRfYlmR&SXFbkn?gFk%1I z(2F{$%JP<2qvrjp^4zU^!{!R{nVT@z zz)RBU3x-Wp@qjwl+Hug=7BdfTRC2@fc0I1`+U2D`{M=l=igwQ4kM*8$sb~J46(Vvm z5hg@pmwn;Bo^f<|EDZ#W+dFsd%s{ZwOGS(;-kV`#1b`OjkuXg zC$j&}$B_^EwYtC6DTD1E1Mz|3>mbh`GrR<{bB*xmsBlxqjaoya-ocnER#UU)RN6&aJ zcL6sv0Gl{ubDi(ptvy@eZ?F26kJHG{7fRn;@$UnTVivou!wOd>!!01)1UZ)Oio1eT zvid$==oj~JSM{%(Tz`Yf9L77oL=VDy#EbZfxGodBATIXz{Dbi`iGP!pIWF#9X?ei# zP@ag>x3a(+e&*6edX$MS2p6)UEcq-^(|P5O$Aa$?qdABT!QP?~ zQ`Va^*KYH1J*aET#?rk*)e*$L6wx>1>*VtHI> z^Pd|xhw6yjeZyxEUm@5F;K_?j>6)r+QEMWRRh(4G0H(CM$zAg3RJa{u%0=0BtNfwi zn`X~zXt72F*5Ym!Xo6o6JDTjL#HIV;KQEZVqA6&;s@*E0u*<;llYei#tS&ud!Rou7 z2kp4wo=!RP+4`=zZ{LSIitnElwB;HtI$jVlRKDrtGd=5Ed>#+6emOn79!)wIJw`3; z5%{eG7jkD!Gldi@U)`LX8@gci-{2g7^z@)+HY!UncVvbBP|Irvzr_R_*m4K>=J#9Z zKDKjN>o*xK+m^A$*r%g)w?{Dppw~CCI$bQW-1HUN zUNsBceIsSiTfplG<<-rRyq~-YBJA^&fjjxGeNS=)!N-&@c61_!GG;!d=e{VHT{K=? zMt!y_mj@C&*zv+2$9c~sz|OL`A3+Kzx+jh{WgYt%dQi$VwPqNl!>QaA%YQ9lx)ne} zW|D!)_KH9CP+1lY)b%Pz>6roMaMwGM$Y+rhv99T-uramLZvR-pQ>0N9}2LzqX&&RS86Q?nE)HXr8mbP+j_*zj-@ql=rPon-e}3aVfk;M3rBS&+e@YBg$LNk4j% zikZ66uuTKRC*{}dH!w%~y%G(u61DjwxL2^JN`ebbj5q^Pg-|B}Ma||Mr{d2i{0m&( zBSsM~pbV9AH4w7fBVgFBu4}z3_js}z%`UjjiVeV0+fZg!iCuA+#-628)q-?sH5tTb zG1;MJ5ac`fIc;8SlU#h3K7D-N<}(agQznCW{Jx#_Uw>w8RJemW{at@;@?C^&4h@_q$uUAf7;2)op**9#k}??HoqRp;h=^bFIY`-*+l9Dk^QM&RvzOUnGyN zTuk*W;JVxjvjR7A+AkHi6I&;(RKPTnjjiLfqeHxe^JZF9B99JNn}c`|Qx?=wv|Ka@ z51Jc2rs%kh;bS-So3EVf&tCTxmK>ymmXgKGumhFUOqE&(+mcmw6v|>>bIAM)CSFPyF#82veyH_Oh?q z`mdv-6}1^-A3)Z5hx^tOO$A(i0LbC9Sq6B0KEeQAEPQi=0nFEaw6WbE#A`3RWi275 z4Wu!iX~$y;kJw&QwD`|>S;r}Ex-aQQ+gPY8&{}paMK3uzyX&Nf9XjJ&>pqq*Evh!} zDskJrPgmeMR@vI8_)88|w$VM2OPF)@!&Xx^&Q7^alcw+OnIoDfIqhwJf)5?AoiNK# z_3ghH4w>l)Bcse;-9Vy6C1 z@Tp`cIUQKgtG#XK0Eq8}@v99%e}KlcT?{Ka7^ZvKPAi2fIcdCBu0 z`+1b`^=U6E0o{-k9{-0(K{pZoq?!TlRduquOHVM9WvjT+tG(xqZ8u9lhP%f2Iv746sAGZ377ac%r3LZG$-N>hN*B^n2{kSu;zatQFNXuMV5c}RX5Ov~^ zpg~iwD9fRCE(5T!aaE{Gh?+_T@2`*Pl(-w3DZ+-0+M$BNnVc`@!A?Uxd@t{I?%zQf z;>UN)DQ&H9=oiyJNn%r}+E2;Iq#(&zhSI$FqReFIJnJtfMpNjAUAa&%alH>L8f z!jW!>$TPAMQv$Jcp55Ya;EZ$F5#gSQH%YI=y2tdw=tZNlbbZmaDwssFaB%l;ZVp#( z`#fU^9g8?Ml6<-xi00Df5{x{ za<^)MeAVQ+OZi{iW}52g8e;C4O~R#inabA&X^h&GKO+}Bx1B`B5-46P*{hQJ77J)? z@IhyjGt7Nh=;LhWuFWv#?b^43O$I@0q1?@L_)G{VyZJBt^~sWf7ZL?JnCFS8ji|Ul zuD#nX;VwA~{AV8VYscLbKxU}TGSh=y|COj)v3xDatg1~LIHRh-4Hd2t-B@gt>3PMNy55D5zYygj+LUhGv5?a zsgcv(BmjtPBppwpSpq7mQ-?92G(gB$X(oU7#nER8=*^@+7V;yLI^adnbymBLx~5FI z#H%CZMv7zOKJ!zO$x(eLRS=V zousMHDU#2+>ZKdcp;?tw%0*K{k?lefnW1b%0wc(4xVg^BemmA*d6-QPFM%hV%F>=n zi=m7iq@Kz=))!YlOmmnk19huIt11qq=vg!E|Mo3iysT|h5OzD?`LRGh=YG=7aM8Hy zo~}5odv*G zJ$aN8ZXacy%x;g3jrZHOl#jk*sGNb!b-5>mm^iMY+dOlBQL+AW1hgWELbGy zY~0p5pKZ`k2NTtaj27_n^F8zP)NKNX*PR-I&f|URGZFi1dN1WJectuE2A!GQ#_Ko@ zwJI`38<_3uB+xeDYrefZS4n3#K=jO?F1l5b{)y0M7r69ovg{663HIzqZD?B68f9z| zbXlQgQDpEhT)_e{D}q7O%w*QlsiScqQH)XG>CG-BkxK-HLh4hRqFPA38xC@j%e-9Yc5T0Y4Kkd8 zo@d$DoWGxIn=+3kqON&5KV+LshR_v$uG#^Oa#)5e$Yy{jI)oHELFJxW2LfhQ+wMdh znOoLU%BcF^Qs%}`y{VZNdp9Jg>=4UEE=fbLMk@6=j4V{HD0dvPE)t!&@oJ-crOG8z zoeHEBVLxQX@VjnsqH)12aguElX_(Z*tUvybB`Q*73^0H!;B)u)ggR9x+~q2CFK=C% zV2B4a;K?O0m-ozx2H`6B4e-NL+}zK{Q%`9q$WVjFh zGXy)wR+!NtYnOh;Ws@Orf|=NklFETQi%k2mc{CX*h(-n!rNkG^5#opIV+_-L3kq2w z!MfMu;{{Zw{Mbw<3yK0u4`U8=($2fL*f;&++dY0wgbG0%9Y zbln7C?iEKxjzomU{PNaR&w@V*Kpf!bf0{aJ>>I*JiNeQ)TVtJ?queV)SnB#C++V0X zN8l-WzqRxJJyA234tS=e53K}axpQZv>|P#7KLr?up)w#Aumcn0E$NWmYcq+Fzz1p< zB>+UxX|(%y>N9*w=&TjU1nz5EiP+U*|XsI%Ji#M~U8I>RjQpK9!5JwIZ zXL6d6^SNnRJMl zmykM}6dP4k*9fm~kvwNOn7vibfywQiQGSh}(r?oeNX6AFh$G64j&jp!L15->ICII0 z)vtTDR$=B&$sAyu6fE~NzmM9tRKXZ>@s$a+NC|(YQe@ogzrN$wIpRwe8Ey}M!@-la+!IiC|)u266kM{Mz2nf!K!+rt;23@lnkzm!@4gM7c7;Ic=}bQ)`SZl{(YZ2A+ZDNXiw&}mW=g~0f@{pqchGAO8Cz9LP;Gnw($c6WZ-*x(1N%4B>Z8x0OSXkNc2P4j&@8hCwjrOGd6D%9q^Rs_n zH8kSG3u{+%;!88}{a2{wP+F0UPL`ejaF}BPi}<+6r`L*KM7&(nnF|Z_QUTFFEQY(I%Jh(9w3{87W($J}XBAR+=%b zz;G-4Em|^J{gUx7GA^9ZZ<3JVBMMjfH&>i~*aM}#jAU-9k43>!-z_?f5^W5kh_zy? z98=f0HwbcxJTxyZy|4Umsd7MDLyE>NsKtQ|WVDe_osc@`I{3_mFW90d`sZ=1WG+xP znk5p)-1v(bN#EXKG;uD_x(Zlbu%z7*0Jym!E;F6@6w8ybV8R;O_rvrnJQ#q72+@p1 zei7ydceIsNSwKg})U{PC_wOAt&t7zzwe=EfjPSRojjg-6vdDc4eEYb`VpyH)m-N1|}RWTWCha z#OL4IYI73&_44xQUg$`*(|ZnX`S7ex=^O5pRML4(ScyDinF#8L6NWaOMRiWID3!z7 zN-MTBsOOp=nS~{~J#(T~mLoh%3au{=47GeNd^RXH80cOzPX{r4Vym z8HEE}3raYR;$MgwZagEA~0x>J6#D4jky2)R3bR9Pp4!NVBHo+PabQ{Q6M0F}d6+5t-opz`{6X}?H;Uyav zr4yvSBPUZ4=NN(uvgAk>3&+^}K0&wQ)Iz^@@CFwMpRXN_tw!o#89;&u@x4E5#0d{~R3iem@l>6~Kh!hREQc@A zMpe9D$sV|2V26+oJMaTBR9|mI0{Nah5arN-!10M*Ki7>?_~=fP!)YgI#%gJel@=89 zw$BMb8^7Y~@K&S`ukY<^!Y<_Jhxr64_r4FxetC^}=AdCG^EI!l%Yf*29RF-#i|+d; z?-0CPlzby+1brsTyBufn1S9^b-GIPx|$jM^P($RE#O}O{HlA1le1pDV9GT zE4xT#!m=SvY>UgPU0&LdMrqr1a+SVg&2ka@xnV%P4&Z66e;iI~b0XO{8h)QZIo8>*}9fw0$=^5>v|3aJ(NTPM;x zVEPN8BT{87=@rc!VMYKD56T!SX7YNfD0GMxt-oMpB1(uS+GAl=E8f+P-j^F;E}rk{ zWxx#S=WZn~@AaUzC-!rz_GKuT4>2xmz~@Ghy2b=opp`#_{0qEymtzyYs#Iedd|TUz zlJqQpW-xnXEc~l~AtQ3=l1}N1&EFDnAze-7fSY1qLj98@T7}%o<^f1g#pXo&xQ{qa zg!$x5&khdp5STHW)5Q}dB( zRMq(t=Jg@>@{;wdynOAYef-{kXQjM@(IgV&75Ah~NXscK*ea;cqxZIiwu{Mx=u0dzE_e!DrZ{C%dX$nCVRdFaY3`emMJ4H1ZK-!(0m>jJ*X&l`oPA z`;TVl=Hn&%vn(EWq6aLOoh65^iRG5dt(n}kI1XcKp@pKUwauHx7D;pBfH4KVeOcSE zfN!{sNMX|aZ}3;iSP%6hr6_wNXlJ(#0QvGk0+-?*+`c9sj=Pj>D83O%>}Jzx|CnWi zYHzWUhJj<(N#$qNh#+dN(G$knTr~jInZXajlXki~7ywOX;-^d+J|xvGPnu(l6tjSy zzf9ow9WNSkrUY(rZknnD*Cd-Oj69cM_kY&=$f{`llW zq*3lVeMd>Zv6NWPSDrs`<1l>T`BsZE11-mqE#=}LaT zR55{dCpGb6s#R;=Vt@L$U?5Bz?*UUq=(V$A^;?DApZeT-+~`$!V5;bcP3c>5go^uB zmh~w~tm_Qc#U2ce7a`X!+S*nI_g)Dr#9Ho*P8s|f|4)JAJ32-hcSq{NpoVzL!-&zb z3k(3)99Fc9+jsh4TeMQ8Nzrw7sUG%Ot!Z?Rk@4 ze92SFRP``ectQ|L=>bB6eV^1@_9drS!<009`{)UC%1C|_!p=Gcw)X{XX-bN}M6ad0 z=ZY}*j88bIPWm_xpVXLeozkLoSMq)#e*ymSBBM0S3)Gh8O&&b%Mvg=b#|#VN&EB#e zT@MsDI$n}^G6qmO>V}`~n>IEO$gD8-NE65HDkMx0BeY>E2SLA^0@QlCOU6B75`Fj( z5`eF2$tRx2)Pi#TU38!w;n!rQ-hq;Qj5C1+=Z*qr>Ps>_Hg5KA>gc%eknkqCzU5KH zi1WBtB=&Zo6wDB{9DIJlqLCbLsDr~h3q!&sGohU}{k_R%2y2=S#cOm9{PrV0waJsZ z4Z#N4=j2iv`>sK;?nrTGj!>X0PO=vWw-17e5?(D|QK=PWbFqKn+a)RzFP^YCl?ls8@Z2KW| z-=ixr$%?5uA*O>_491eynmA-b6^~&nS|GV4Q;(K10GQhp&`Cu3h z8#Zk@6VuHTu|RPN(e=#CJkz3*O!@5;Ip?EipS#6vZ6fR`=!XrTHUs>O`%cw}nV`RP z#FUW#1H@mVX=hivVoqA1eC4SNj>}blx_f)K!1LfgKE9U}@lk5_vSi@}LQjC3wXPHP zZhBP4!!&`%88KyVQ>bl9n)2LNsn!bGH0qo^Az=-|P_S-NQBEMC?THf)GyGsA2p29r z`#$M$ym#km@cnNRm}{}qY6;$n0skRo-Io!AV$`GP$nO!i92U~Crv#W1Qk$Wtm2`Le zQ0;a%k|pX1QzS@n%s;ctF&ai(twKWpCD^0y-Fl+c!dA)X@<)uP=Nb^pC4>iL+Z~!V z0Ov9_Q+@bsQI5pi+Ns$T@>(qDoG9iV_|Hj5LImB&u0cFM%mi$$o{DxycNJ`MWGThL zC3_r+b!RXYsEIQEJB=bWSe{(%OoM4s!imT|=Ti27RE??8*n>EZjq^vaNiM*W;;o#M z88=KW4IM zJ0hmN)b{G$MUPPZb7X3!&Kp~OetU{fMR2g$fVEby_BaSetRZDeB-07UXLX08caFP_`8LBf!E4E4czlA} zY!=W$6S^Hz63~duSlyH4{b?=to&~w%!xO8$6(gI4?miILsrXr)NaE|9!xIC0Wotj0 z`-+k{7t@i}uBxGf!U=Q-Myz7QW?rfTu{8W2N1sQrtK-rrFt>ydS*fFG>9|;nl@7SQ zM^|n!t91xfVPVgZ&c{Zb+T8uGgz}E1S{`3+Z&qWo`FC2yZHh-P24oz&w$Oh4oPzu| zD$Mq<$!q$4WXoql$Ekx1Ua$CuZDZbq>;8ld66|+T!VqSJKDNkcwn?)nH)4j0 z3hLQf6z($R7ijEA=A4Fd?3$aj{aBTcl;m zlX7qa-J{Lh%tBwfZKeQpmOy=!}4xxK;K$VbhV#Pu*u&nIe`fmw3^b4e}r z+knsyl*36Hme>;(`nIpPuk~N<^xf>>XIS)rF*K9)#Cy^qcB01`SL2j=&U;tlVc$XW zOmm$dhm-9Pu*$KX|p=G-b8#Ut51H7=y z0gQa9j{E!UzrR213oYW(1{9e9+Pq9skJ;;afOCAI(51L4C(R*FyPt1|q9J#i;xzG8 zV&Dw+RN&Pe-=(dHZ$LcER`Y5pcBRUOilpgnDoSE9Fyt!9eJCWp^k{1O&a0VmE-lP5 zuLL&R3~+RysBIS=F5nTlm4j>syHlv{-;yMskbe}34pH`*x4W-#+e5~e;5oQa6(VF$ z!?oG{vW{Wod5E1;zX|p|TaJ~rZ1uJx}x(%2+Qa3A^+GGFh#1C;1-(}u*cefo| z>rRNH_XQcd4hu8^&^&{!qlUdXM7j(!;<)?NDr_UY^*4sXJxz@?z0a)w!!yS%zV*Q5 zNMR2^s6@G0dqZMN0s;b1*vHi$YoX2MwlKsro^e ze$8J4PT}kUUNm1_%(hM+6o*2@-IAw=>g9pmhQ76;Aje7IEoT-bTMXfdrg|h!^?Pq~ zDZxF)cg%a%^S!T<0&SbF(^sYYafd|D+*|t1n>wks2N(St%qb{3|5*@F@77x~Y}<_2 zL+xZHVSqV~!kIEQmn|3nt3mFLO7D2#WT%tk4VQS;Zc56Y_@0R4#Nqw(@0IxYggP_V zeC5Rh`Mlb-Q(~Sg!QRNVxD-x_jzqWjFS}5ZBLl+-R6lg<6xy(qaI6=%L3i^p>qP&F0jllg*5eIcg3hJRS)bqz@ z$R}?7f)$QELMeB!XLY{MH)w96M$ktWva<=ExV>JN1?K2zRjfwfn6-89^+r{ueNxl* zDHXrtLMv^CA$P~2mi_x9%47%X0>1CyZS)9|Duv(}6oj(Kt3i8H<+Q>p8vWurJY8_Y z!GavI2(dx|Pw2N{C6ljk{b`0o}#sPO;4Lew_Gw$Rl zJ4Nz^R%u^-Ra@ABqqe`TmO`%Z4ONhCAGv$zHopG1OojjKiuqflb6#6H4FTs8;Be-zquC(q2 zu9<^sAxoP-`bJ=R;E70vM=+k54rRuo*at-U;yoh^H1EM1JFR1`(4wwZoWG{sX**cC z7tguXXwBa_xY{{*vO9IS^Ejzy^obW`b9+djyFvr{%fF6G=MgKKXcx0<^h{V==B&D{p&W`emgynk^!=#y9uAGMx!B_aTZG6Tyn~_7fWD_dFNIDew=z0;G}E5imp@+n9Ro}cYXL@h zU+AH&@)M$bLXWpVq)9dhW1?=D@eGCwMEjvX=JZ5Y_Vaq~MmbZ-mO@6Xb;mWLY}bsH z>B6Hw&`*-Pd~p$zH3et{j@RGFexiGX{05mjVKf2f%V`+s$%%bz8R#{&m(rL#p-xNB zLy$)A${S1{vZsv_iAxlYC+;=I4hd*g?i2c#B8h}&yM10*G$RE2VL+!(7&#|C&ov>X zBL9Y)ao!>MWbLV+5so9nh#nTnO)Xmu0WL*?``xj~OepPME}Q}{7Y8(k&Y~`@T>*&< zdl?=sU2;ofK7vA`NP?Iz2CVK;a2Pv=I@MI<=;?jCKTCXTXSwi&Io+XsOWE9d4OC^{ zYF}mMpiZf0uktz&+GB(b^Id<;rg+rKX2|oHio28)8bkeAh*J0B`lmn)@Ia(bsBT$5 zfw3Wex1ZV=!ZB8qCv03sf2_2GKn@XsqvI+VdLO(F=?NeX*ovl1Q9LNebwYph=?Emc zS%(8*B_1qGuM%=DT?Mdz@QHQ#s>Q<86S}u;`}>5gD?UKApNiXzU0iy(88)sffTnE=GMv zw~92GR|gD=O^dx_^_Nog>)82RR#@fKVaHm11!v+-Rin2PFDo0@*7ro$uEjE%)!OCY13C}j=o!ReO zqDS}TIgNTek31ROds7qQBkq|%8hwGn_8kJT$_Q`}`KVvdhz1qV|F!^mu=`c1v>xI= z6MISWJ%?A>$P4sc4Hl!n9(G66c0BX8BYKU|OC8}y)iYv>^VI%0lNXK+fRYygcpELa zDM@o}^f4hy7rSR9M@-#>zVV}|O7mmM838xt2+oeN4pH)AZBEj5D>>MhM$yi(1 z*{7_=$m=T&z4(4LgF^tK!ZoRjEm;oRAD9x#p-Pv*BifK#G{)0etKic&nzrR@JMe`Y zZ*u}LB}@Xd3{>+at%MZ8S3eL=FLKBgJ`lzYNU@v_G(&;45K4VYWE4~g6{)rE!N|tc ztf7YyFo&v7?O4iTpI=B7iFDJmN-uzp#CTAka-}bXmrDP`k*UpDRA|B+{<}zDCyLLi zcfo6bz;!BZla+v4J3pH!8QTg}R!&EH_kr-m$ICRpuKUK49Lg9&cB=mp)>}bB<|cUA zg>c&?7Ss(YQKSaLC=2!4KRddN$y#=BJTV%}pXw-cJ8W4YAxQ=b*{MSMWa@)>Iaz9H z5AQ;;pgdtG`xtm%POxcYD!v1I_BS+)TUplty?ZMnwtCoQ9O#CUo4YsCT{Kn2_M6c= zY{C|Da&tJQBj~~6RbyUY;5=(I}FvR-kngg^^|izJL3G?W?OaB-7HtiFETO;BNd{B4+RMG-hJH=0Zg z7lVAv+v?#xhM$ztWt-jlt89C?OC7gBKfT)|R&c~+fImAM+!ns*1Kp5G-`3WZ24A9y zR6u9Gswt?K52{i9xfRRAE*AyZy{+Th$?1hvb7|As-NN~HPUxrN<|VV~vlGH~WN0j8 zyci=p?2B>UvplnP2w20)@^+g{iC2a(Bf?x)E6*3xc3Lxi9zk7ZNk;LUy<>3BqPq%1}9&m1KsfsrFENj)I1Q4A&r261SC8`kW* z6*~?o09Fb^PDJnqMzmizG;Jx1SNSa1vgKWO@em{E(*eYk5MXs9NsS9L{MS%ay#I^t zJe0(4*fXdc;05lrB<|#ctt2)ir}H3TP#oZaJuX}V=!K$JkaZEhdo9iZ3>Xm{(3gE; zP;Ok1EdAwo_VasOa2J3j7mC1y4if9z?v+0Ft9>t1&75)|Z-&wQ=Qf262F9BqXLCr< zhP+fD`Ti8keIj=tcFY~vgmdV$E81Eh6uOa!nW3bC^3DhgWxQp_)8dgX3cOOe_C}^i zzJR;5Qv(NWM6GI^nV+N;RII4IY*3!woZ^$_u(+futnRtGx^`NxE5m&7OjR?|N`M&# z`axNdsBhlH87bhM6)xE6toFx8g!68*%g&v!ZuvrW0m1dd_G8o;TG#y1PRbvDhS~n{Q3nG=zgfsJv5dSn)W<o;BX)@|x*P6YlefkW#1~Fd0H^f-Fh~V0D^xX`K%1 zj~6>1+D+rZCi?!c-Sm#tx(oFj_~Tt+7YU4- zwL4QeN7(7l``zFC-WF&6tDw~Q^hl7=aa||25=9|*P5#tmV4=Bl8&XEQd5EtINozwt z$!P%>-hRIT&YHRKKoHG5)W8`r>7#w^pD1xy@}qtrs`I7O3I{0WE$2B6)9%vZvWP1j z;+0JB2C$(C5pqK|u&gy(eJuhM>Vs#EqK!N`5!@6JuU{9aV_K+;Z3FoR-@f#9gztN9 zPBMZjA&AU5eMCfB0U<^dc?wX&g91%hjY_G_%($p1R#XZ4`dH(*GB7Mn2S4S3V+JjK zqoJ$O?-MGc57^o`$VDjix(O5^28BMxG+k49(4{f_#HZo zZyzq???cWMvTB{${)-e7+A*3kynDpnfYw!Jkd zkaYA)7=+0wS_lc*M~u7Se}5j-#|8%0yWBGRv7klN%}EH*RqtP$0Ak%S%@#t@7nxn1;wCyCOtX=A1Ax9PMoN-~&e)<% zrF#ZB3Bm9Zkqx!OS~*&uYIPobS(y}+H~n+)IAvAnNTEV-qO%hi_ZRpse~o#zOU=`q z>J??PtGc9|{Be8D>5;{qebWp!>rhhc6*D`rgTKlI*cJ9O|6U&lSrCP`2pVOk&W)?% zWkfsTYlIDN2=9hZmj}`4;A`YvHB4)KPL)Lx_W9; z0g-3sKfA87>R1{Dz>zSE`RZObi6xD>&()C7%rXbakK^#hQW5UVvg<35 zol=L&8E%f}MaX88mhxJ9^H?*wmCj$aqbR@(Q`{QJ4&w3xDzOg{lQZ@CAq~QW2};(C z;ElTEN+D|uC!8T_(){#r#r zIxTA>ak2vE>s-%(A&}ODh$O$e01o%muN2|qwZ0T>d2<&0$(>8lJ)^#w$jl_Zn1;A| zr$JW_4#=zG%>oEuiA_?;g4-Ag_xxspJHShKr|lj2=w5RUC%O$s`j&ne7eK3=oAFsf zBijo3qKB*28$(qinZe(#cfwF~4Ij)WI~}4wYJqYN%eWyM^wm@tm^H{EK4qbdUG0vU zXJtP6Lo$SM;iE*E5D)N_t8I5q{8;J1L1S_{?PyN!kXaTS5&z#%=9y6I6zc~fbHj=8 z9?PcLms0LfzypWp?g6`IU@U=^Jc|}W&V={466KzSo?&<)D)Tv<^{!4{m#2rlw_eBO zeEm&zVLb(urnvJD+mUkxC#%F@V}ATY@KEM`z!k79xw_rgvZgnthw=N>kn1*nAD5VD zwW#T3ILtnDnYqF-Q=vAiOG)%U4ZOJjOlZe|Y>SU(amIU=Y9XeN#J$FXZ8AW$_iD`WMey`QTH=b0n>Ge`3u#B>^@S~QTSpUrN@{p&n zt5Gs+CCV4dEqEl5KW}Ov4JS)Z|AwIE=m~sn3~xAOi=HS{fYTkEzU3{(Qs-Hn7q}MO zzIhWolz)Sh8kgh5F5t~@!ipL7+*miQEp@i$`P6XTKl+Kjv=F$sRk9tv*OljftJPvK zE;0Efyx3w##An%G$dk&SS06i>cTP#~l2WSA!B@Fe6=<{NTSt|>S}-6zRE%2EKCb-Y zxZ*T_nhOQhVw-C1 zj|;H|Rdk0}?wI3<@DSGyUR3N!7HKe81*ukw(QJ+|%gL}TO5n;1PzZXPXEfhLZ==Yn z;c_o&<7_{pB|hGzlOh_04gOkBLWhww?EUMGW-1wjnnf7E;E&8iwCosy7cN}qS<^y~ zd);9EcBANyLJEM$znsL(NTm?dr15G{IPFYSN)4w$bW#HkOry^Spm}dU-n3SJ^xG9iUWi`^SjH@m&&*g)xr5 zyed`9t^-$mEBBRE9m&)1`nwvD?>tvmGG5my;Rum%-@Tx|N&m4WTs5YqHt<#BmPWS1PWj9u^NLaw4srLw7j4_2GYnGaCARFu-S?fMbzh1Nb;H= zyjIs>TNFTrNm@b?rYxNOG_AIh*ux@&c=02yAl;x%`#0Eu8Zo3hhH?qQmJ_8e@3G>` z+5T7PVXYt>WA9>WasW?zdtBPvV~C142&Bx%9F%kc3YW`}3?2no`RK^Z9%EJv2eMZd zU>9Q#9plj_vw2yLg|QnGfDwL)Quw?#7L~99{PBn^;I`De{=a%`6 zNs~^M<8;jw$E2gSh5MToXz!;MaqHLUfib;GoqPU~TvSfnjeI%xU6aGHfjRwe&t4j2 zL}aB*!sI*&>9%nJ1!+!a&#Gk(3LemgXg5Kc5_@&x)pUeWz;hWMNbHP`_>{47pfR2# z>yE0Ao#@1ebRtj=bW>43ZP6;n;R^@nL%AVop|Q_Rtkz9hQFff0Icen-9$Y9BvzwbX z86t8jgrfTS#4`9D(RtrTFCyNioc%LzgX*?kPi)NqGav*rPEBPAZ2+<_2Ckg*is9?< zYK&^UkpmMNz#?-mOqqAr4E(vS9b?iR3#JZS=mB3|Zk0pt*Of3S z*jZ&MM-(1vYY)1&Y5Pq1Dayy$o8`7*XL=&#-4(fi?v5_1n8e>d75@49o{;(*XtA0Hg zOgD19?6SHiqgRP77swDjWbh7rlkpB}awHVkutf($3^f&4WW|cau z4^x>SowO~`QJvzc!>Bosu5+1D*J=t(yr@%{wj=VHY>J6trPqBAwPefc-#PQi8uYzJ zhRA1W(l;Py-Fajv=0QIuqXggmuc4=@sZt6*t$MrbHIfQ?W5>@=Q&98nm=j zdZ29T88RpqW3lAyyK8T%*@0_T1_cL24miXPb0PF@TOqHOS!21=R*GTm9FL27dFQF7 zKCWbDmoFF+dViHL)>P8th1J6n`dOnJBbkZV%0GlPVl*|cd5N36Qv&23QU~kUoy_6( z-p!X$uytnY7=jeyj29SkWt*Wtm*9KkR+&(>hRT{iZH>P!BNFw_?gxNk0(y_ebL2;` z{RXx37-H96nL8ho*i|TQz|~$Uy%%4Z1%Wbc;^351EVaU>rroY*Q`B80(#8o;dx%QS zYuitYJ~QE^k0Raqc_RhbmP$geZ`TW}zkt;MWc%X>!0LYw=xqSlT?e42?oUVUZvd#x z_o=1#Ln8027GK(xtGr4Rat zkIeu$`*f;<3Fu!VA3B---gYor-CQTz0ob|GC*Ug}@GV;GE|l192+?alf_iK(&4Dj? zZm&$6z1egxVx`v}jI~`}CQ})0M001~1tr5!=%#>`>ctx{xws&TUxHPI;*6Fflm+7? zUP&{Iu}_Q%)M4-#u_2jxiU9dj>^%AcPCEKCo}tR9w*Wr0n@qRp=s#GG_{V_s3Bml9 z?lHw&uKWz=THGe~zhIetxeSPrZwh?7Lek5GHgzbhcu{Y7m< zs)7zP>FAI7sp%vS6v~0Vp5V;PMo#!TIe4(@aZ~h|jqfeVSI6`pB98>8J;04B|^kO0B+3?IYrFZDyoRLQ&n)yox`CRkvQZ z$T;AM6IMBHn!`1Bywwk~BLK7S1b`^2z)lgJwzgRzdbWO9n{C^Qy?_gV;olB5^F%{o zOomuGCR;1UO3l}WN(@8^po>#M5DH+poJ2i<-+!f;mT&WTN^!%F@zqlw^# z4MVl=ABkGW@>^mA*|gmYuO(gx1P5S`cyhKD*|%A%MZ%a!n5Wn{vy)AGlRnzcFASK- zkuyYoiH2~@2$SqyX=Vt$z>H03$6+eyC?`M%(Z?ynM^K?T;GhR>jE^;OmC3{7=k86s z6T;Sn`nCN=%?#+b)Y4Jny-=gZxvk@@Dr;EDnr$gym9}8DU|iK{$z^Rj&EZbE^cFhD37mLZ>&5{x)w^@2{;$q4)Bw+l z;ggjO=FPzLO?ugxi1-+s^AT^%;q63BTyPZV|1u7)o$#6A0up0=hd zHe41z!w#MQlDeH5d~DK#iJK?_h4>ve|J|7V;B<=ngXkq*5^wEuY}Jc%LxSeZtCWS+ z0U>8@U~8())|mxkN|mQin#B_ZtOcev36d0{#b=j0&wH+LH?OduX-fDzH<&xHjbi{w z-q9K8VZuJ3(Wus990;wwd8;|i>wHdh3!kav7rGx0i=6y4j0l@1Igzwe(76PvH zz+hKEx6&ie^`Ik~$|Bz_ITfZlXCMXgMSldby%AQ&8$J=J-PSO`Vt4R{l zg92ki)+?Q9*m>r7uU{}Di$gF6R&vf7EI&&;2A(OCYGzT`i6&>o&Z}2!E}Rg7AU6@J zyJco}f%_u7J75^_9|53pDcgJ(?7+b0O3GSnKYE~CdDGp;M(;H~6l%NKksyW&U zF_9)jdz--uXd(=pq861D%;W~7a9iW^C+~^NBHq}u3^7hCyfYG4I$~H``kISwt1Kp% zDJoDzT?CA1G7-K+aTqo06w-Y*4N^Nz&oe_^W|nt>Y$W<3Qfg{2bgEc4I|vXll+uPQ zT0S-lbU?~2&1pM>f}{sBln|=GN<132dC60W(1al&l9T*FzBW&cVgY~aZ=F0ZieM5> zsh^~x&i4BRVM^8~znKy*a-sSKN9$yQ%lxV6LdX;X^V^>=O&#=q-zJ|-dN}>Ki0(V$ zCVqzLAbD8^dA;G&RT_4jU@i!S(3unmV?sj0O$wGV2h&(AR8o;C=?xJXAz{HKV`Fnr7@KK0}CJWVMWd)TruaZvj%j=Hf4B=r)&jl&0=TB+r&vgl8;%8QM z^%Sa*u`t8)s42*Nmdj;USW2Rigs-t_LMGt;D;X~f*RJ0U&UGFq_=tGBR5Grbbd?0m zqiU{GQsvNt)zI}-Q6g%ZoSi;mIQ?t9!QaTvBk9z7J=@^0#&J91MuniK?RU~Ye+sgZ zB;!I26D3fxw2Jc%$f^0W%%g7ZR>PoKY!k5>*RJ=DUa8WCa2h!Z<|BfhsihCwls z>x>+OcUd5vRb>ru@rjs$a^BdwqNhaclnquzM|(>4UDXN>Gyd7lo<+>1rO#5U>U*B) zgLO;ML3-}SXT0ZDYWY;|TyS65-^mAuPwam)8nO(w$uTPAP)Y^VLx-EMM%n+v@1hda zGenPInYF+TN+D#8c(TbS_;bcK@<9yKAD9kja z!7R2vG2O_V)zhA|ur7J12SrJegqK_eMC16#Ieimp@h$aUwlQy8!l67?tW;%jmi`>a zmyMVGw2Jne<~pM}%9X>4)oAbxhnUXXI0bypAc|AK!8kuuW-KDJ9$x zgx)ZRbyG7kW>-7gjZigjzS$U24~~nA%WV&5j0-S(1vW|s2;>JmikBc;+Bk<4>90*I zi=wDL&QA_MlChf%P5%fJ=l*k>|J1LsXfSOFY0{9)&VaQio}~$)t46pJq618{yZ3iR zAIQ`B{Dm2grYfJ45z}1*{}$X~>aX>9jXiSu=0}_rod7nWc*0|%kvbzslOoU%zUzNZ-Pj=Wn42E&v+$pK`OVbTa;}J>5D&HEA zj-jyHb7IfI;2vFgK(`qnW+5cf)mGIB_5H!(^72c>QCEZ)p75$1=HanO5AIP2;&2pIvAnZ(Pm^C)K|y>dMr^eybZ z5*3_$v;kS3ctncwkQlaCdvFlaze0r4-zz9`urZe1695~zL#2-q*#eVCqwZMJ9-)9Xe#yaXxd>;o zjrQlshkaqGltYiuqqLr%5O^?-d-kmeFZHIZx~%!KSW6J+$QLzxr$`)aI*+MWy1}$; zDcVlAlpa{y@ra-$1Cj{Q{<+2!*p%5D3j|pH+1c!YBQnHMcJGj~ae}X)48>qwX^n*= zLn5Wl5?-*=Q6!j)hq?e4D=}p#@{!3PJX`dmS_DA2j~pJq+W{w8*TR%TicG=3{oB6@ zLZE_eQ6xDhp+Xe5x&Y)_Ods>CiitFLM#&gQ!muHoMNzgMmwjce9)AH=onWeExI&V09JoKc6_BiohWbB%Vu~N3 z08l!))X9O1L7t@Fk|qmnjF*yny^z1o@+jt4nRfx`c`Wom0skD3n~2LqSx{!$qa&3o z_1}aB=n6-D6iopg39D%ax@a=UXiqihwrZu_ut82SqZU(amlI((L#ZRaXy^PR$_>WO@OM#CzBh|)>0V1a`AL*9 zda)}t?+a4wm-iyR8l9-pjhi=8=4PFooF0DKJwNE4fB472u{j@h>5@)(3wv_ibmir7 zei8hQ%U~3-Hy_X6c7ODY1!!&B=RthJclpj6;CV_DWA3Jae_4^7nQL2nC27LF)jWks zv4Z}^+ZD=*Y^$4PG)!+qITtaiW`8ow^PQ@yco~M}TZxjKrdfKu0O&qEtm~n4A0GPpOflee57G$cVk5f1l$Mph2NVV1 zMZenxy&pig+y6H}`(9Sg+v)xA(tcs$(`wLr*&n`et+paqA@)%PRE|T-Rj35~=XZDB zohP%(MpDi8AnqG9-=Bfm_-HH8jRNt`?)ie^wEdkUP1aW{TrCRq%U%psS?YNuW9OL5 zCgXBMHfB!^sEJgT`?D+g9!t2-qVw{`XOQ)C1SAvw)3EZKdZ=8Hrh(3SCyP|AObfN+ zt9j|H-1`jk&)5G(BUhE&9?f!IhaB+Bicz(Ehbg{et{_yx>@AP2z{P4r^s$JP-z_D80 zzHIMg_w4MK52yPV7jF-b4q8rlhlfs@LU>2(t*i>_0C-*wdU0KS zZGH9fS?ts5{>QK&ZPQ2f{m;hc+V)od{%3P-eR==$6wfnoDp!DYV8}QV7wX)(NYD^d zmfTv0h9)J$kX*sA=R?Lq)RQ$;G>lVKFRd0QIAR0f{X6Y@Gc};ekag5O1HXuPH58mp z;+P~%alZPkGJaCdaKa5hG7G}1ROaQ!(d3J?uX)+wMWjW%d~Nub>KF`MwxgNjm-dEK zZsUX^v&D{JtStnKufk8%aA>6xzsf$`QwYJNPbilJrYh2+sLNlV!2tPe2YAQiOaZ`O zXFdep!M`Rj^vW3n0}QIX7OXj@Xgo!Uu;_=yJFu-4c z*Y%2u>HZI$%nw4RvzvtBE#Olp6QRR67I61EzaZJRD9?BgLQw>+kPjym3D-Q+wNcuS zVic-;fx`Vt##eG6LECck1QZ zlAShRUZqzs*`hOMlkX7i4+Lo%04Loj(mk|dRz)e#LF?sEOf1X4HKDi{fi0kqfcL*V z;9YpLZlU}JPviEe;8Ufb7h_U6m!@5;r_$l&=tGTxdP!T6v4YG6ky@s!KMJ;#&Ia7~ zZuO%+N0iMmaxvv~2=Ka$_fm-o4#(Wsuo@}Z2GXfey5(flP6Y`P7N$f}fdEBN0s4gqo~-q1+KxtgEpc`mGn$%H zi=599VVo=0mNv8#p_|O$npc$7D!8aj1*p%3=c?bGH((_s3U z6A~v_NImJTk~of&E9h4sQ+&>2M-dq2VYCVGb zcQuDFXBQx6NH+_dHnuBz%S}shPSF>JB56&q*h&1EFA^?*hTwz*U{{6dMoGoAj$&9g zr8P)0qJa0O8+-@=n8Pgnu251{nq$SPB9H-QMNCYBeN2;yxbFLD5)4r#fpwHFB@x=- zyt0nH09saRx@ct#o&I#ae^P8Wzi`vSRSPvO)M}C9K^mkBQ$U$?zN@7xn4XEE8Gg~f z90!w&3~`yUdf!)rNu^Sl>KAD&;s#*rx`(-{*)T`~Rw{>*jhqjJB@@JHq9CYqqd?AF z8HxR|8HrtOi^mwU(fMuM27f22${!zZZ%HzSOvM;InS?6b4DEr#D4vJ_DJuc?`QcB9 zkVd?GQ4jR)(x^uHj}NOgY_`=<%0s3eHKu*GG|yV`N#sK&KjkiE@-2eG8u8t34-*!PJ(JFjMhIB<-Q%n0o4$$hVQUT^EnKmHOH`{fl z=jKc)%cwmW0Ow|eL`XWBI?Bj_rEby{`#XXtI zh+%enQC?vZiEGshO*44i5xhuHfT?@tRL%i0y?iBHZ~)y!b+pP0aZXi{0 zVo9AFbQQp9>8dNKCD3tk6%w%|)XkUb@)Fz&uYa-qx|U6JN=TjbA-9*+)Eq{*&hxYD zG6%nSY3j6!l%Z$6G1^ew`;#O=5ql3~QQv^3P*k16Bw{#{Cx(bYoG+&U6C+WcrV2FS z9v3Tf@3c;eH_ndcJ4sI#15VN8YOW2_sbVEWk<}mPb%S7C0adJOJ0d4&h`ziR5l_7T z`Jd-2|G9HJ?Ie(bbdeYXEVm?)V14 zJ2TG-@Z#U;3ms9H19XMb(1S)`-7Nf*h;v>H;|Fj=8G=DCjpAa*z%!B`3NK$7nfmHh zCcV_z1=GZ)&c)pGz>4$~?|C-H%aE)O@4|b{u@$SBc-1?kfNOW7h0f}_$}K;Wm8@>M z_|ATFS7LhewQbgx&*Gj|_Mb5cS_l8Av;SP*+IaOUZ~wWzwXux<@gz@)hSZc(neg5f znSedYIeX6t=j-uz@)xzl+nNv_R$d7%IOP*fDE9`>6s}yUZmH4zJ}){QJ|{JWWytzE08vwB2T{)&f(PWr>b zf>opnz93ThuT)zq8P*&lE+mcv!p*FZO-=_PERJ(U+|pM>sB-wDTgDfMX3|_LRgZZm z9U()|LX)}AIKpgK{JH9G#!%>#wq=en>ZOIl5pGo@{2C9K4}QUPKm4W>Iz6r0^v z;aIdaA~z9-I{m8jELxMt3w3>7sw*85HHV|^&!x1XHhZa!de5|uaP}`vDH7AT++e7y z>d+C?x*?gQR=U!rO-Yy4%1(8rTJZ3&Q&U*lko*@N!Hrkjb%iBpUI!FH2RtFHp38L@ z+j(v+660zr%Ykd^;VklCS91_wP#Q(Hj3jLXmjkgKL00&!iWQpv>Yk>5S|Jjxu6t?d z+8idGRi=;ui`P7e^9yX#wW%XLo zqVmp5_IO!^Qkql^=B#vS!Em>xPi4&&X_TfD;Ik*<1<{eG68$D3*(dP2LWP_L>#2$n zpx4e{{|Z!Np!)^%5%aUs&ag!u_|;7viUV0ZiAeL1Hwt)E1PDb_lk7#QB%p#KWTZp; zLZxvmjIVdDfG)GZk|xtG0;ORxjM*&EbmDH4E)_#(tqR{;jn(xvP!+DJZZ;qF5NrYf zil%(nnftL?5+*LX>LM%6IWC{Fo7|Siki7VEpF$0`PVt~|7yJ$mzx=|t1RFU+5{1sC zojQ{^hQvRoKg}{aSGg>re>s>iyS?C$^b;^g$h-oe?~!F~ZIVL4R>i*D`x!THZ0igk|;&(6JE(WBj; z4vt(SFBy=`X=>xc8Sum`{|Tn?T?>6*eA+$wc#wzS?oJ)Wz0y-OMZKXI(w&E7c!Z`X zEQLycFT(WU-Nn(try@@7?n-Gx``$`t?(K)8{e#oOd|2n52wy`niEuRRr`O)OhE5O8 zK72ggt3a$9R?RfpusUa0lJbuc{?`P(#UT=I3dJFPJU;yIj|U`dG1OYRJTF}+nQ%sbHfofqK;9`zME zAzp~HLbGy0*@znRLAfg54i{tq_B1?DRfmh?fYM&(=6^DHzfta|X1y(f`>}W$eSh}7 z@HI>2*fs zPX3)X5q8JovRv6MuXzcs%X7XAI(O=|(unGk@cmCy9QrzWvy`78MooMm-paHPJ4L|Ue05eUv7&e^Y0NG5l3Uo zoAMk?%bW7=hVNe9lt0N$dAiYRQ}D=qHy_UqPHPh5)aB|a0DpS$-=bYUmG%(wR;@owlMP2L&oGtykBIoocx12xpyo$6fR>Rd989Zf_HBx7S(~_eAs7 zKqdEt^H!~zYcOs0aO&>!mP)!mfs!s=A{BbNMk0-KHP!8y$5Y!ibqaMFV^s!fY_08E zdWA0aeM_%UyR{a|vJBKkY0H+*qhRSg`u(W67VAFRUeDzqAPdlP)wDR5mu-8+QMm5|jQL-l$NXW7S9P}LxN zp7Uf@71a~XULz&d6V6_H3$j{dFU`pQ%o?gk2$NB6-&Kr|(Nf{F40^Ft_bh{6%%JMY z09}ltXBit~85^T&V=Pj&)3E7f^o%(IsTru4TfyUKYpLE@s&|&^ohMN57<+Fa${myR zKaFZ9$Embd?6^6dc@+bdP}LkRbxHwe$;VXa)S!6wvtu@8&Xdeo14Yi0%~*RSP7R8e zDx5#B3g@w6WjQu_98QKfnpt_HsxMVHOVy2YQ9Y&NM)&%St8Fxxi&5I>)?6x^3LMR? zu4&QkBJ??F`%k0IDerD^x*XNV+?pH>>-S`&qhbDrbTlbQi}%9%iaMdC5@xA{St?-` zpoB@u!$MRr8O?qg1&qaWv{t{kIF@-u`3i_C$#b0;?{8M4=T)?*g)y6&C1v0FteR!g|L>X6Xf~NSdieW>AULC>%0H zg8`EqG2vLmW>lwn&+uBi4*0a9Bd3r!eV<_)6o~vzQ%gnZ}$7P=+EOr65;~t5+OBy@a3n z>$)cGxi~`tl)MDbjgmWW@+MGjRqOMz*6{$WkOZuZAm#Fps^R@cN#v~-UJr1Hyj6z` zm^I(!gjfSxvQ!GF?`QS6mjBwh8HE%B)86c7&|9y{mL7gZ` z*lrk)U@hKyGvcdOrBgSgTt;49>bd!)Y^hZb)Un1^IJN#_H{M$Xp41M= zyVCyPSu7}XuKvhQRG2Q5?ES)*N!E0;MWV|;=@LnS{Mrj-PC({d9y*?EZguF_V;GC| zvB>37=-ON*ZA!mQ&o=kEN~@rVix^M2x^&lHdvVPN$D$726U7|NaqP}Wd&|z;1&zNl z@^YEz@CXs0wNe6<>m19%GlQoLds7Y9+1^Qk3w45~na$akgV~&JjrMM#Q&Q6f#PZ*CC-4$419p=Er57|Hs;YY|Y=nBxu{g z$9>s>VS1)}I{C1Fad?u}ao$Z2Fbadhy|j2MCy+A8CSnHCroO(lgu9pOk$#)E@P9tL ziSquw$N5{76-7$vut#mUjfcZbJk)#&oInywOKRTyw0z%*fQa92NO zaefM%3*!Cx$zrx}qn_OpDylg4V{Rwr!Cu(vofq_yr8vVP6jyksC<;)5l5$q@-P?=P zgX8^!)5GI;+8V@R(KBK!-s$)2S6f%~+f8)GWd`-a`cYOX+-RmyYYrfmXMbO|S~Y^b z-J_$QcK7~KW0hogqu@GG=P?JV$xz#lEkBw%c+T>-HKkWY@7`XV?4JL8cCo*EzUvtF z$`GZ*-$=h5VQ;k;LpG9n^on+mKD;Z~!ZvC*BtthHl%tI;np{;G=j$i@RXl=yTsygRbV^7w@*zhaESev;i=%Ajy4>*4sDB&L$cJb|@QSr(|k0HgeVc|lV~YgjWSIzYZPyV zSgGO_z&oyqQevhTVhhPp+=2QDf(%JYsSc{0Hvs+1A$-fL#l1oWA&g76LS?*pSB%1M z_A`dDvtv=W{9;o6sRt^59XCBR^UYaXzKs!Dy#ZSJoQMkP=e#vmwq!6vW8=&cKX7<-eA@k4b8Mf`8n8iHaNrB@_c~Xd`(QG!8e!JZ)jas zy$+t|7f$1W6>T;P_L}xqan>)s^0a8SC2NvYtb3dW`` z1e)~7G@VSdfuMgmzq8vI0WY5VUT(#kW}ImSCNG!vLWonb#OFVb&FNop0-_Ks>NZw{ zu3B4z&qI^3jvy@`>~qR=u9LgF{tqA`Y{cbcWk6z2xC%Gm9P-}TnZ*|dPR*d-U zUuo%8#(8m@ZZsZW>Kpr&##R%8xXk{e@%W4YY)>)5MusFB6&c@9V74tIhDuSO&6&_!PU^=k?qzx0z?j z2o>cf3j#g$Ht(a6(447qH(+67M5txvqJL)?ND+`0Ez{)`Fx(E8`BqYyuuOKaT1-pV zwX2Cj8INVQnwG6XdH)-A=>xD31uirNQi`$XVX5`*4Eaxx3c|FxBs%f zwf*|l+Scn=uh;*wwz0Xfw)PjW_P9fr&xA6V{AF$KzVd_nP9BG7n_NL(>D#1hf@vNk zKBgDh!K(d1ZXUTzPDzMPCn2J%jus{q9i;R&EiOdHYa9hTzHXLPwM^hZt2qCyhA&I&j}u2BLsYsAh3IK2!_01?)CJb3?sfMZqabu0)|BV*IO! zxWWGjF=e96onbm%yf!MIzSF}n$efM|e26`SKNR{XAb zFVrm}76b3ZJUgcA9ZfNXiAM3b@Y`3jt>Z;v;0bs2>oY zO2T!a>D9$>f&EcnG&-0tWnV@m(|}RxbR7=L~atIFI({{}{%O+v_TRpyvEvd%c<0|88$?Z!ORNr+6IabSuqV ziLHb4WKQki{&0#~hwVj6#zAAli2>REem+tRKWIu!hZ_@;nf8CH+n@UVzrB{<{~OzD zo6G(G6i-f$&-3W*wsH&O@wxcwZ;4Da+PL%Y)5m(+?0>^v`8P_UZc}Q9<#!BE+=@&z`(Ye~^86OD0jk=R0*>ESJHa=NH!;VR~Iqya6nk z7|LVicwCg;L^RzUb0S>*YrNcd4PA~B@OXiSj)2A1#W2hw6^fC8R*Be}kOng3U6#Ei zk0^?+gRFvJ;|e~a3Rjr zgrYMB85&H&GsISlH^C0Cpw%WWP?CU~oB)ht+E3Ym{dARm7qW#zx{#$E6uz{2xjE`0 zua{%w@7UgK)`Y?dBh-f>j)umBW|P;UkdN zEf$bfwg0&s?iezEbX0%IqH7&Frf$0g*%^U5V+mwvsNyM7^~;HHmIzH8X47UN7hZ_lwtOeQ30inRmIF_EUG#?@Vnoh<_JLKMSc{x~S#G#XMzP}3J z;#_xbVbtngho0%3EByC2SA{FFhdE&`Ze~Yx zyu}9x%mp7@m4G-MjjeVZ?!f$9SWF;7b2_A-Pl>UNFx6~;=-g!AHH_0!fTN3|Y39JI zx1YP2cI)qYa)o>r_V~X^gcuS=Q_4lDn+6&!OT<-1H+J)f&&AF%gV#-YV z@MH%ZlL!^W?AAi4>WWWFYAY@x%BQjaTSefHvzYixCh!rTR`frHEiFIfQ$znZw>H*t z^nZP8ZOQ*V$ul4QuTUN4NwSM<@&%QZQM6MM?w?75f9G;*ctV0*)jUeRib@#9aZa)> z?>QHhOxq`mJyoIr;-8Ev#oop+fW z>+73q1^d6(YfJmTCwZh3jNUn=iD7D-?-qK4pGy<-DAKn*(014RuDXl4?6+Hf!mIyW z$CZWvcG)Io0qz>J7zyNCTCB-HKEY6p0YUu?e#7SYFFZNCk3^Rn3+UPEPs);*Y+n4y&izi`N$mJtGon&9^A*<^_ za~3!UFF7 z!TIUo-kBR-8H1&6N0oc&l!=0_tc5mo6}bxy2=f^XGk9wA7;ZZ^$=m~*Z#=Tx)a6lC z17^ixo$kuBG7bq;aY>+@TUp;|0NvC`Z=c^ZM(+1M8^<}6NX+FFMQj`oX!F4rvZcwV zkTY$*an=A)Ok^g8XXZM4QX~@@D=ZSN-h<2%h3%c}9?$ngP@`{~a3zv>4hq zF_W_!43xLr+-_&Y&zFn;_iFw1lK*>(XHNc4 ztpTZQ?S*)Z*n?eRW3!<}Fne7r;h2T#D_lPMWX#!`9U$+R z?Vf<@39y7q={^`FWDHd6DLWOD;Or@GrL&gW-P4wc@!L)@UW|v&)cI|!<@hc*Ofjx7 zqCezR*WIR?+%jdeOVK>A^oreV(u?|<671_I$7QRf*sJFC?FEzqnxFIr-M99_m78l!P+&0)pT!Xm^pt#h1r>Df4=SWq zW@c41+km1VCOFbzK4!GjYzH)s7)quvJx=v+`y_Q-9X0QQ`U#BDIUXZ2NsoH#Wo2h0 z66qs&M}#I9Z>4E#9da}XymZ#{2sPo8GVpo%*@n(SD#P;Y>gkJ}hN_miFrOf?CUG=O zm!;YgSLk`gyKC?8f+zo-zKhG)1g# z1FM3r)Udj}vq>y80qRaDbA+pFoR(B<(%A`Qqx0Lic_+GpvvJO*YLF{=*tCWs!9syx zI3`%yiahMXk=68KP#_8h!Btof&{aWvODl;>IbMaj${SzvUjMH1NLMSd?$lSrP6;4G zUeA<4K;@8QImc=*q|%-{M!}t?cnTz+L=o~OJGj2Ku_gO> zkab3k#c2(V2-P2AzUpa=$1>19{hKNplHtXGBxA@VvBnX2@%cH!V{{P`ABI4rcz|p| ze+ADIG$ssPz#vFKH~l%GDB(Z;H9?etmFFCkiv$f&f|Au2YD|Ko`EsiValw>jgnZW5 z{$`cpWIb1em-;bRjH1XUYBCrgPMoYsAcKKjOiajyxDu(FEAcl7(G-RB&HBy; zxJI}1;&&3EH{)CJ@6P(#+8E#n`V3FeoAm&U;g<`Fe@AcD*M=!N#xMpFRg-v_zyN{g z*#xKW(lw9<9#micmVS8g0{9`J>6-`L180IzVl)woR14YQeq75C3IO!^g3ngAz)<>i z_&bg*{6&AWzFmxQbP=Lx$VOoOI`_>d?GxUY^i&E92NcF z;1p6n!7;O5f9V<@L{*gyvf&1p3I!$r zzn~EWo zxDuM!eqZcRabd8zwq{J_Ls{z-!NecU(6Uj1POdHnIAkbMlV?p9H8EusCXtWKZC3$i zh^MIUfKqIisXTJPdT;%2g%Sp`Kk98|ZnDZ6d+Gjj?ZwKfYG!#)a(bNzo@Lq`Xpl5W zKZ@)WyOb1d3PW)UZLF<<&9x1%9@LfGTw4d5YqiB-bPHaM%Sv6{$|WUnBH#HWVhIV& zDliB%3e)HRbWa3luqTH3d=gWZAUIZy$it|v4N|t; z?&?fN>PWMdh2#o`Js&a_W-bo_@I*IWE>vvRK6v55I6j+PeM~}c$Mas~UWolNUGJyd z5$!kOp885crhPG5XBAs8-EoN5$~U4f5a_-5TMh%_9%bf%UH?e?3QuB8b(nm3_@;58 zu0l$1!6$MG8BF>w_z(TJwW#m_tY|iiS412j@c%&XhyUrV{O8VZ^xmiR{2%2d`~7|#Z{3sO@SY5(_c!Qjy!w3AqB8)XFELK^$*i9L zu!({-m;~J98|JOG*H}moTUT-e_KcD2usc&Nv?59)wlbGWC?T<_U>8Ms_0d`U> z`lq`mGOe!i4(A0tj&Za*g*eP?EzCEi4wlW@hF?BLFxR^EuC)Ex%;RhCc9xpnr+r$| ze;snVl?_0h|Nr`S-v4iFYpMTzn#VBwTrskF+YEY4<3b#Oj@hl>2u-p`N%PcE7MbX^27?4OW82RE1ZUtS@Lw_PJUKkRz5o*uIw2Dvj+W5SUb z$h_=#L>hF}Ecet-@Bb4##UUD^U5DiazagKYAbsOgoc>KkKAQArKyIQNm;}2ghgkzU`z-q;dKXFK`ByIW zD{GVt3H;^Z$)5e<WUFwC8ZRH234G zFSAc7G@cbYz&FMd^Al4hE8k-SD6BVBfik}F+KmYq*dKVHud)}|$ z8Sp%SqX7H-01=ucpCkzEo*YW;QXa3Ly);!j^ zP8?ag*Re9Ql8#NgVQpH?2`SHFxthV}UpwD3tJL=Vzv8nQwKD&&z5m}V#DCsg-&n@~ zd6LISHB^j(Vb2gSfM?4eVF~eGd%AI7)B$w(mm*y`n_@ZBf8K_tTF4H%$1Hd+O{zUajZt|2LNT|DWi|@qdnd zPU1dX*n{cVqYKaQ^bHtnJVx54mWi6oz@=tqs|x*PCKUSWouA=``Ar%>N0{EEIi0*! zup-PbodxyndV@qzGpBw^^$@mqr48>9fx%BOgprRD&xFbRPz6?Kl+&=;@0wq-kzf6Uy*YVWU~>!2CLsaZjcKVmc_Ka|&8Ok5r(g!xIIt@nU`hq@>CMaQAHyp~x$j%Nn?U!U+X_ten;t@Vw({J;5nZAt&1kTI8poMVor+YAwa0){X;7E4v2l{dM@h^0RT43_OTE}G+2UiTR~#djVE;6q zBe8n_6e_v6nWj~nq<}PGDGvdq1aoNzR!7ys1=g{I6BLKg z7mm2r#MN}%9%yrW%nUZCDzgBd-<##p-0M?p)~(t{$jaH86qK&nk{!?zp6V>2v!@09 zCK7=zQ-3a>{L_m6XON-6B&02kKx+8^R~uW!_}@$Z|7jj^yGNNw{TO1(fJYhifOn?m z`V29Tze{I6%Y3QtoHTivP-N7~nPSZ4R6c^V+^d{Jzq*=tD~EAR4QV8U>$|w1d`v42 zEdg*g1N)HL*2C(hOF_QCvo4>9t0sJw$D{#k9 zv%UB$FkfiNy5T6yAAkH2@~`P++?P%;7HU@F1V;=!@0}|I^nQXAiF|TiKXsZBblF=K zfY1@#c(qL@V-t|Hi_8MOiDQlfQ52hI0Sh!{m5wG)>*V33!Uq_z-7TYNIHG9S zp+?FATR5U=ltpN09N+}@|*U$pb`SiLYAnK3{fTf z%gj<-v`<+Ilu!7yk^dN9uHyf>)4ryU+WY@P{I9Lo8(T~H?II5e(VLOGEfilJJeE(pc2v zQlw27$tHAQexC&B&GVI;A!4B0m6?RP38FEHDB^Eo&LV?uS7%@Pk2U@y!H9AS{rql{ zgb!fj?|)qf_W<5pgBSh(3&79k=O_I;#r$nNyjWGeeFolh-lGTJyaC?E+L{NX!h1=| zzvX8}`rj%PbRGTQ+Rp3$wqLzk(*GxUOaXZQSSr=W$egiWo-R7HKAFD1gC-5DSQ-yn z-G94+NY5BD*%5G9$3u39OE$k&21_;yI^!}M-*d62)6moEw-Gv8z0skl)z35#wQhi7 zeW5Fz|Nfwd*o^eQRVcVR`oHnIVE_AieR==$B+vIt{~t5X+@DA^NrSSvLZC@FzJB;K zF(1GFQEBkkeCDA4jYEOe^8Xuo`u}QU>HqOG&mW%t&l8XA&vXpT6|SruLbD)Z{M0E4 zBi7DiiD_{-%U)$cw$A5hF`t>sZ$CEsF#W=EV;=K?L!9}^} z%}g4;6H5Mc1ZlSYztH%9n_Fvz_+PKLm;3)oo~HMI4Wq!EkYFxOYaXKTwV1A!G)1du zi4m)6UfLUhQWak^7HhC|egCt)b9&}g2~7X?m?@1mjn z+lqDlHoao0QV3Vu#vzGFkosn*)5Lf)f1DFI7~lvq71BiiDslMM zku6&G!AOV_(+LPZi>*RmpNG%ZO@ei@nfFpSvy8kl>C{+zVxZ*|9bq8 z#KVXrXvRM3DhNdROA~*) z`-9#%n7NP2{lC7w{_0h3|F6H^*jna)f0E~!HWmjzqi_s_M@6Ue12~K*gJCFh_S~Q= zkUH0Jk|4P?@LARUbwwE}>O+_$!~SZg^XwUThZs0TF#(6sfOI;SmzO??C<#%AA1i>f zVgLpKP&EN>h#D;oF&j;;dYJTi`L0}>M%_sqK!$j`ozBG>Lf~`yO+v*R(_&sgd@6p3 z|6TAx{Z(O1uXr*L5Pu@9QG~MlLJUEnLZ`-7C^{zq%MUWG5q~R5naH0l**IY+&4m!L@Od=F93LvG#$B=2Vo}oryS9C*74E5x=;$Z^C%7PKl{!Bp0 z5d2_*AVN1_41XgDm`bffr?abG$PUzN{wo?HuoB{H1g7h~jov2c{vAv=dTYJ4RVy16 zWE_+P0ZajmIymAZfec@T2wWuyUUMi^dl5ncz%h&_Fa*#SGLr!R88|0ki#HEecB24n z_SQFg>#H1pn5J+ke-0`)+TY)R4PI<5YvC1dq0{*%nSc?T%3%iq=ynGr@x{+uGD(1# zu!%reHak!vRfUHk0|W7*EjfrF>ug(gRt@21Z`}m&wXE}2?3YM+iQLMGCLEEnqTHh| z1qhd>_@ByXmmr2*reG)mH;{q^@qH3>7_lpy>he8sIN+lPH+;4t!a$s$DL;|JTR;&L z%cZ2Rmmnf)x;mItRQ$Q!R;R{^GB-qC^nTYB6X%Y+(>y2@=JB2ueS0TF0`Z*y12-31(j3Ydri~-a| z3vfj^oCUISxZszM5zIkWgbRKVa7`j7WRCz>rdqx<`^mwTm2$T}A6(gadBzgR(D1gT zDZamSQ9xDMs@B2f9q`y~jJzG-of2`Fe-vzt2Og^66>x=kiXuemNkXnrWodAviULtq zQ?s)5FX7@W(>uhx9dHLcF2QjiP4AMIz~jQ7w*xlU*4DrS@Nk3~hT%R6;q4jnNfc0T z2W+ps1fKFN$=+`Aw*!d7NrKKt38Ets2HuYF2)qPpV{=ZR!3gicsV>@RgtHFt;Zner zQ<|3pW-o}G{=Eb)FST#>cEHx_jh8^a5r!D)tAY8OFBQ3i5==xF+U>Lqw0j`5v19;F zR23kjm~&2pQap@6Oaf{GHqQ>qpZOICJLE+{s=}>n9b5*)zeY(9lcvyF5NY*qY2^%} zR8w3BQI2Jw0WJoIb+zunhbc-D94IDQ%Wk3pz~B(vjEG?N<%y4}$$`hQrk5+1!l(EO zaV}5rotLY85VM_o<{;>(cIX9(4rIigmA#1x_mjU?dWMyC*bjyDe}QTt>zMcRTlBHwYvFGgBT6FbajNm4|Rn zI#JBUP|ENfMl#)mIduYWCFL(UCo%83*J%?f=Fze4(_{-yxaK+D9ocQl=22OJnHCxq z1dQ`Tm3(kBxTG$mlI_@=P)5dDYf{_#V&L-YXwAJ0n#L+hfu`AF~`cW$Ru~n=Wb~(4mG{)Vb}-g>z~3qg0xlfNDgZ zl%5Au z&)^b_JQ-Aj#o;;;_64nhv)~f@sanZaz!$G0Y(7ii`s@jDyoyl?F7{xb!~Y7d<1`>gzYG9quqNQ3}Mz?dXR4jDxv{7r9BbdyyQ0c%RmkIrn2 zmy}-!Tv2{()lVQ z!g}4&-pR-Qdo(7=ZEAX$hSKYYc#N$Ba89~u8u-@G&0`Xz_pc&69|;tPEcC#I@^isu zSnbxWuk*BM6X@n#U)pH7!KK`;wsuaH_BC+LgD*kv4kOI=QiiRZU%<@Z>+~(W;9}NnU_#?~hP031vEtYJS1gwO#*6L0T28NE-?^f{_->W|E!_xEw)d z%hvrzii#mc`P!n4ACCcK&?-L{|DF%xUcvXBYkn`Q0#+{Lsej3j?n@&w{pG5FOMGho z6yT!x_bRx~@b5NUDbD;5;&Gw;<@(w?ypVY`gX8BLl=EmBQJs9+6D*!=LC&K?_ttuy z74ek|etT)dRhY**e9dK4;120qZ0{bN80*RiLtK7`{P`5Y;Fkn5^da(5V?tJht4ek? z;9{S$h|rc4n}cf}`enj3z!41b@9yKM46dwFzWn89;Mu{vlMbaFnrO;)kGg<}QwZqf3IsSwuUQJ) zV9rfhfD1>GO?DFgdvIuzR7{0gohtj1i+S=iBJwns=7n5T)j_HOj=E#Xh%*B4I8Mk^ zjB7xWF_+rrI3%iu!|C!7=kqPwpfmke9Q7R73_KfbFh6fzqkjyXxHauy8@V$KU1~@wk^gi z78ea5bJ$@tfy}hYxCfV46MB0^T5Ebr;d0tx)WgN#&?BcgWXWvz%XQp1j2ow zr2JeI3`Nt1$BH^wQ8dK~ky7~-CRi%_V@RnqeAb3z^9sb#R5+f#O-O;a&BNt{XpoRG zfEg;~I_QBjB=5U+PYyvqdE@PEiz~q;6f^gEJIS;Rn-bB zNG7smo8Vp^~ne~FbiBtws|DDs+E8iT<=_P6=dkSbl($3#5nDkSLfs!MLFjC zz0!}G7Hle%8eqJtyV0ts`ZtIhRCh7k{kY-DK>0GRCMsFm0tyyjxL@9YsMue zh2d~A#C7*Q60Q?jR2>!}f&8?9A&G|ac2;i&U4E{0&za~(w|Doc*ITx#jLL=Mg2mzL zZ2Y9$k0DC(V~ zSwa^MR~^Fc!G2m)siaD6au*6$4P$t3vVf^1p)SlMgGVz2&LMG1xYB@GQO$M3y`}>% zD2Nk0hRJQE1;V^=xx8UZ;1WemJwd9D(ieH+8|^8$qDk0D7A^pCQssW(0;Jskq6c=v zklav!u+W^s2*4zPw_F^!lkti&XO9Ok76QTbkF-nKZHQK10%F&pED_h@E5ba*m$6Dj z5CmDFH#g-r^NQ9A+k!JV+TT3^aKc6?V%RS=Q=&>;yg^sTd1f05)#D6hCap zPr@5rw9LpjycFSHUII446bPwIy4;TtxJI|KU{@6ESpuzSKyH%en@eD=#5LnNyKAok`(p=`m@+72*N%Z~YMcOh~Cf50q*~6$K++R+}50F&&*GPS6=6 zG5CBSEi)@Mf(+7Y+Si*Jbz>4_*@^p*G3bxFBNzoCO1deOT{J}z>pu(707twMW#Zze z3!|VLCx|i<>soYGBWNzm6e%WFT?T0cX$vFQ1(2)!<-aEIw)#TI2|}s&Z4rcO@7sHD zd5v%IFu_4z9Hg>#J9sL*Zu9WiH#|;w)_WV6 z;i_|_o%7B|!&PfMn!%ODudHl+FJ{qgSmgEOZ_R0P$bh;d0w>4}88OSN%>!%%6z!9;)}I5KHC zFa2VJiHPvRl=0a$x}}oA16PRMAQYK$6^w#DNq~+ne1qA@)TxsMj3*((afrkLCG_i3 zX4mo-P8=8nv@au=(Y_HJYYcFdo}KqL zgf3_V8&EU_6P!dUXAbaC3LQcPq@bW9G6{n;-c)+EkR8hxqGTfty9+9Dn|E1+O6cci zhOGfPb3r6$A)WtA1yQyjgeRd~xiRwb0Hfe#aoH9pQx2gkh5&HHOY>^fxXhtF@PW&P zH<(H}BmKfB`(}jwk<}BXAelr$bP=6iA(7s_2M*Z_3SdYHhzXTj;CZ^7=-k@zV= z#E}q|BYVX+xHxuU?*=k|^d7MhG1mP(kZ*XW6B(;5Kn(dT+nmbXkoLeYe0G1?J>UDe zh}3`psJ`h-_?L1yoE;q;oB+&ZIpOQc2bFDpF+~xeAwsd3mF+c7QArfY@rqV2onz7m zLQX+{CYPQGWGI+M90}wKBJ(oSiBnEst_T~6{`eoWAUHuGfx*YqBbyk%<;UnG2{}`p z=}U70rK~Hiw^0X$sM-mX?s*O#=cgK_GOoz8?p2n$ETuzwfO~sl;Sw1fCvieh5&2sP>*gT`wRPdp(2>dN zIMYOy=!TjdbB02*`B4Wd&N>6Gvpub&zI4K+lQjr6n!=0LOPl5Ai$_v~OZOpiK3rz~ zNM0*dR9TX70W6TU?Hc7=aFJw))9Jhihk*u%BtsZsDpOqW<(nNy#zd+U**u9G*|*ig zH6UR&%zJP|hS-N8kgv1zQ6p_AvQ*}D<}-*mwQvQP@)fl&*6StTcRs^Y&XO_$;FRD% zInjqmuQz$?fPx%!t@NR_aEbT^AF+%YM+IQ~jf{pXVs>kaZ37~#qd3GL4`)%**1~1^ za@+&GeygJI7_Ea#E&@Epev}J|zD>xOOFZ<9B@>@b5))X7L47sK2v89wnXv@`)fFWw zMa*y!3psCK?jGKZ*3f0+b8=+gElb7^(({mhLJ1x?Z-xqIH?IL$C_GQ`Kwn7k?ye`C zNW~cVB8YWcYsoT%I|Z#8NvlXdtE8>8bUOhj4`W{JvH`6UzM6sCRsb{LvuM2m+S`=a zYb~(=Py;~}Kkkz-6p7kRNLMD22p!_;Pr1^`=A{A~Q$P`0RotA2B%oz1<+>;qX`hYb ze(w6YdQ{n+f|f1MM`qZg;vVB-g;=pTGQ|l-08x?t)Cofwj`aE#h!QrXSh2x#_y{&b zSvnXUFh)Vad776gCGd4H7dp981tyU=07j^CF$n$pgqd?H7ea#4n92=@SvDxA@IRmQ zZ$SHp(*Ts?QN*4u?%&-jO2P?Kc|SM=B0GpIE<#IuK|cR(%c5Kq9TKJlShe%!fzK&L z7sTk?#67(joq9d2=Pkcdb0cTD56ag(^eau+8vkK3pP8|GV|q`b;4zExkdAuQ!Rt1!I`EDx7va^0E=>Y zlM}5Gm~GFq3^KQ5G~aoafoz^;D@400(bYfR$(CW%t(g06Pqu7eZZ7}nPc{=Gm$;WZ z*>v%*cCJYPd=;b6eXCUx`)HFWT#_Vwn%+KDR zp8!8W0g4!gA?-E3n9AUqP?Y=wRfo`%a9yKYtu9UB5)in7;|)hpp~K_98UU8T6+=pI zND@@RHWOUBVF{Q1RRf!}dgmLh$waoDN71l3kxqY)hI4Cx+F)+WQ7D0GGPj~+w{C_F zxEjpuY;f6gE8(h}VY%ciC)!@MbL~CY)!(aqYN0Sl4YY+JriG%I3ihEuF>>IP`VOSY z4ph-P*Ma+rrBMC2;F9`x_uZKaw4NVuS?dc))}rAmcLOQltHz~aF-aISTv4e%gORjC z0Vb%2WZYiLD9FOeH|08&t*egckS?e}=`6`>D1buA3SO$K6~3M?-Rq9SaE_?kmbe^V z850Xp_5^?nrdhzATtUBLMVhedBmhw$z&-tGcdx=$d(O2f;7VEj%AocVE(%UbD52wj zT;WJ$g4Op@%7jK=jHz5gX%+dK|NKc-q-lsUE#r#Jy^)VVp6m!vClnnhTFYB0Ip7pa z20=ok8MU7Kluq*xpexv^IQ^aa`gg(AgtJXjypTtun0%NLvOrYP!&lKq((d?*;8eBe zSOqbS59(70s8w)6f-_|guhLMg2+_t`P%k<1A||@0AV$d;QyF|3QgDMp>80&9xUk^* zNYQQ_?fqfw!q&btN{zTnM2H@(}jCG+T z6)Q4d(_41QSS7Gj3riRJwE?t}DoBpZ{+` zaFxU@EP;!`q046N9$Ze>dt0-{Ep));_82RL%WxX&+2!Z)6|>>0@XT_+Rq32HZx9J` z6NLl@`=-&gD>PqOygNz6vT$6&&_=h~HI0`tqE-!a1ze7hbl-OTIma;E-h%hwG^J)_ zP>aFEPy3;p;N640Nx~U2PQn^^=FXfNw;ZAzv2#OI?x@R;=t&aFcpZKI=R)8r5IYxK zRSvuN;PQ3v_2#V3x@B+`+;&}XRXOa=>8x7@S0PBN3$7}M-8l(xy2o__<8nRI2ix#5 zjS7)n1m)o3_*%IaS#p}=H1`NdYRK&tt zaFU@(-C|NdfvI1>J-A%24*A*!uJS}@_uxd^Cz{=tw+=4*W5g2dx8&2Da8ZAR+$yJg za3;Q)DjU_Wgp{9&He4gdD#H1-f{TSJij`x16*E&^g~Y#3wHW&9EQM(!X?F?}d7|YYp9Rdiqs;}M_B!uM+jwIVWBUubmo;ePM zgD7+th&CHt?XGT87?B(Mf8TPSYy z&>EEHFym&w)wl2t2Vzmyp~YaXl7grm%sWt8H31)43})@mtdf!M4+V2kVeUoP{E>@- zsE#WFXuw=1)U;tUk2!2!OYyZuVCsG(getFVfw{SP2ijC6>RT&P%@=Lfk!I>(E_;($ z0A|*b@vwnSw>bQQdM(SqT&Bw{0kg@h8)qgj@C|7+b17g=FrW9HFMJtT0A@k+3s#&| zfL)AD4epxTe>X5`mYw9h;301d@IHzn^lW#Uk#WotW0YQ-M#0HrNs}g+O3*xOD>`>$ zs~nUg6Nagc3EEVvFdfn)8PV_(Tx1u6d54p}C|~GJB@s#+owjBQU4p7R9ic@U;SEWR z5?BLLWriR^DTJ4dWh;ZZOjB$DGiAj!&o5DG4WKZUD$hN%e3O!&+-Be+3kC7wbi~Uv zB1KNe<2Xd=jHd}Y?CJ*@r__YFBj0QR^NoUX1u%J8RN*Z65-9Jexh~2ViGDPu9GI#B z)*|BSdRHrM`W-}%MYYEtx!iaautmkkAGzLmjfF3$KK{rB$E!zSC6kVuw5obU48bl> zGP53}t;eSYen>a;<`d|1_1p1=yjF!R!Rv{HNo-Ph1BOGMrdaBAJxO>a_*=MC8~#5@ zejOJ!p2;4p8*=16T!T?r|B6`)ux}y>M}F-@2_Y+07GA}*LX#p(B8@G&!P;r81^!qw z`_@4mx!I*5h`NA&M2*(=wCCcm4RbKS;z0Jcnc32bE7UiCW+@Le7JfsHoI(lE1dPUz z!O1*JAPDVvS4;=y#l=Y)_!|u`uW25=;?s;4@hDEJ+{CPSq#BaYZ7G4V1_|Y3p7Cf? zMWz6Lu2?SM4TVWy`$2z?M4SnEgiHy7W~nlQX0`g6z45$_jSZp_mYe+scS$M#c^*WO z>09$K*u%if5x-)dHyjRUbeech#gKtVR0$*bFG3QyC9F9L>>1tFh8lbrM7 zyv&mU>3p-5DcN{i47%NKwy^uQ(Du;|^neM@<=HtG<>*`jI)JzyPc1;stw>J&K#0`D z1p?)hPCGnucpR-`XB-ZPRV?u_Y1B3Yw78YUJkJ)naUAn>-7s`Wt(GE_LsZK>HCm}` z6oKbyU?jD+m0FX7g*?xh$T<4jR%aX7?Q8UwSn`P~IfA920B-=*3E-!DMe}YFk2IRP zZaY=cdZ1qYL~Em*(dd1e%m!HB*qPG0%q*R?)>Q7XYFUcBD<(8&(Y(fW09-uGMM0%_ z{=od{Yx_OaqMyH(l>r2zQ?MA%@Pd<}E(|sdw8yrsa_v@BAMSWVJ=<`H3_}qQp+wBk zlsp>(@o;p}oMlox_@3v4dh{vYlIq3=ImS{-&H}v;z}=|) z9~rZ2GKI)ixA0O_sm7#K(}4U97@zDY_>>WKeDw*;5}GQTAccMter-J2X*f~yIOK1q zk(;h?jf_H0U_H3-J3@XFJROi5PjmIpp5^czCtRm9@SI_F7t3Rr{qcb3V^YtzKD=bC9r_Hr) zn0QKR3<4CPwc*$9PDODJqR-6z9qMx0pVZiL6!(O5>p%n&HPG0)$oQh5nvYBHPT6=s zw9#K*L?NHL`H^;2>x+~X$I*c3zqR&mzp*I}2F|lT{Fv~|*n;N&UdG|2kKx~g+W7t5 zUjM}^bEt1Lvv_@WescQ$-J1_TzWMpgY!LV{Ho&R$di(zG$MyDa`DLsC4{6feS-5dE z;&OT&T*Sqs9F>CQAy4Jm2Qia9?3c zC0}5_moyP3uo?X~^iV7<@|guMVj;&UAOX$_F^i1%gcjof^n#Np7MFo{JVyU494JQY z2v!hu(z27obq2q-O`=u@W7`R#QDG+ofd}zrkl=ahi=&fc%sHfd1yveBiAETjOF0@M zfWirz(gFEa)7gxd`2`Gud)=ko&solYV`1S-YQF7`a+-z{Cc2;TUw$0jc47SPXZaua zTcQ0sB_dacki_Acwxks3I4q&t>~1Q@;W*d+!BBb_a55wBem?ski85zdLPInjf3y3& z+I@Ex6;XjlV}}@6<@b0*U|3%Z%1F&l zhhoA7b0Lz#$8{bT1xr1=5HVLJ=tg14gs8*=1>_Yb<YOUMUH>u>%l(_P$ zBgs80^Blc0C7a1|p0PZck!hUgunwDl$5B;mvOFBik-=yd{{v-{0lDgXmvI^mY|asH z$_gs4O?h|WrH5`^M(nl+i72a~y5<^baR;Q+?|6E!%jcpdZVbqc?-BAb2!dbDT~<9Q zCJWNyqtcSb&zu!ZXmR(GlEk9Gb7JaA^4uY-9Z9q0#YEkpia=P@c7D9M=Z7pkKlPM= z?FY!i1EfK9P7hh(^hnu2#n1)hXQ46!2u393;9NVNPKu>{McfQfD)WLqQ-SBHWHa(Q zm&%LWR!pfP2Y1t4#i@A`1LdKJr_4FQa1`KZM>}v8FUc~dRL!BvUiDik=5%qA(nt+Y zrHc7hT!nD^d>4FcjELU_;wlW(eg+`yt)3iBs%K};wm75$`GZ|fg?2H#_dq5Zv5-nW z<04Gowu)3qmP5LGIB+D1FKwm?ym@sl2XOAOa7YwG*;1NG=90&>urzC4R|-z1GzAAY zNnj4Ev9UiiilpkoMRAc!qO^(AfCFCELyiWEc@hHI!jLQ{vh1PCi<+q|2V_$UikoD^ zsQ7sQk9j^Ib5EIvm0e$}`Lhc@8U+$$#k&Y`K}Su`8Kkatyql5if+0T)~J6CV@jdyZW>c$&^PD>#8gRARhnORnHGy$OJ$0 zL#qEZE|bI=WJ*81&sh43r>Pi_9@#Jq;_4vS|DV;C$5Fym%iRSnL!PGZ=%=^wm`MgR zApQOQ-lCy3-t8v$p#6*9-pd!k-VS-u+kg2Y*q8q}c=;kYkpDP*`64*nfu=9}J!dpI zrwD1nqLDSm;b((QXf^9$s3u_*A>u*&i_lzGTGm%d8JC2x-HiXF&3BJQb!v*op& ztCnN6LB7TFxuc)7qVFmW3${-{PE`xfJ&-@-G@HZ)TQs19EqO+3Bk6;CvBF4r;}Ur? z0x8LTnn{HcHEptWhAKJF$=`B*Eo^@MpO3d2lj2YV9L3HVOHGaE7bK1Xm440|Gb1BS(hW#w)E000F zAMm>ceN>+NN*}|nyxM6)fw8N26B$tvhfp?KHH0sY%3{(oqit-i+Ip{_+hKKjYAn=y zo;0!33Nc8JZu+YUqY=yX4I+PjWq)mqHl_Kc5;}oS+#~ajege%V1G2nOU1RauHQd5j zQ<{B;K2`#4N;+Xp$kQtZHFgS)=2tKR6k2tWU%~a^7<_A2jXqh19d!CZzhi$d5&`?8 zU!SU9|5L92=oQP0N&{yv04hHEO&&s(S+}|dmWKH(1928R&%05JMChqHK@fB*Kfqi2 z!XN+6X4T)Ws)EJ+?ugbKqNZ?}^n8odt(N?<|HYK8dZyYhs%p9EC_w{CS+Y>)0$?>% zI;dYeWuY0;8%|g<)xoAiG|JJUk`+?aw^GgnNlFVl{zK?AP!w;@GHW`Hr*s@*bsR`m z_$Z1_BjxrY;i>UN!bAHuiy=OXg(<*_?_$N*SaBsOvs>;fYFjTKc?b$CBB9|W{9VS; zz-&1-*&r+~CQ5xrjmX4^vdiIF1R`5E&e! z;Y1D7LUthDDUh-Iz&tI5aVB~6fNcKffBvUdjUZX!fNXx^pGNw-GLzaqgfCTYoXR5* zYyh}rcxY!lUw0#0HqPCvg(K+A)OWt_*l)cQbP-YO+qUR-s25%SvNr22n}|R&?$K;% zt>Nwa*}AKhp0_lXV^J^Ey#1#$g12lo^iW3d9Wvq9>`IE{x{jhyeg#Qf6p3Q6A)l&p zd=c3ajG+V7cHnuiqlOUh;wn|_x8Z;4XhJ@+9o+x0AP~1McB&^9oYi7h|60e0;e9YI z3nn#h;0F{^8eB_3#jtGhnuq0-rRYqs0s~Lkr@|K36|C^f%<^wv|Kb}Y!dsDF;qZVC zPR6WIG3U5yb+w?QqLx6N5|KNYjoh?%$As6yt^2JiVplBT8Oudib~p$7rdrQ=bphLf z_X{s!8^PrRZVh}r<~jN0=-pct!{=P0YI(p{+>L&~2MQQ;-s1%P*nF6PuY=h_yB=0D zwbH&{*O=u8Au>Zczc|kvAGms5s?`2;s2-g_ifc|p!7|xOp79;9qNfaxiyEPFycOAK zR${3{j_Rqyp4!rH)l?C>vnlT2bav$+T|u?ddxAy=X`-Zu6bF5mv);qLyk=RN=M;BepHKZrvA`R+mJKOaRb+NaTr-DuBVsPYts zAj{VHT=ix>M$d-`{IvB}t4#JSym@PVRtYSzom*~`oL!U&&662Gfu19p-wgHqLw$;s zqiZd>s%G!%S8!krkn;mSyU6LtQWyBODRElfW{^xGhE$FkVYDElGEO3V64E}yputGC z8n_N_^2|8yZDJwXtK@GVpdQ}wO;0Ils$g{5M`HA0PJL<5(q&s5A-TQXtPc=c*?O-Oq|3;tRTk2-ei+i8Hwgn*mz2g z)5PY!2Nx~nvE#^yDkjEZs#?JSuBQ1yAcq498YaNeN`Zes3fA~a)!GD;kNm+{=+I#^ z=iKqytU^$|iRXa;Ze$1rO+m3`khB4SaWNnl4{8VIS5RCVm5{imhcGK6M_)olZe2j2 z7J!&j3cgS}jKnR7o|09AED9^redHhTLZl62CDi-hLEmgBWW-Zei4DM8i(j!^s$h5* zLTlLe6rC3Nn8b>pc3O`V*w<(-Coo(8gUw;=+(T+#;5N+BxiFT58A%yYIJ%9d4a9?9 z5FYDDFW3gg7QCR!vZyNknj3vE)J~2NMaiOoypi`9pdOab(ukB9KIB)BqZRH-{6poy zDP7g1URMhL54D8q?hX+V|BD%ejdoIxg$7URy4TuZsTmX*22e0Q_<oa)Zc{D_*g1h-^FQJZ4t{?IG-WG>gPY}S9DgEPn!dL^?>Hqqk1*+Sdq>rW z1sA~%x)~0-T_*@e5BA;m4rbl1xO5g=N@3c267I3UiWCdtZyd*XyhExPT2#*f5=jh5 zC%uT%Pk2?pKXnXSQ0PURN;1aMrz$Y5@?+DBRzlK?UT#o;-ZfBkC7m%P;Ui*c#v zvHa@E`MF&0=PoP4u9~tdd!ARu>6$*bb^rza+Jj(=TkS+qDL({(J*NIf!l9O3F#&bN zS-J1~YkD)h( z5}Pf4c2@A5UNCZ;7AzmrkddudXUE&-NVgq+N?x5ElPH(sK7m+>oE3S@t}G4dsXiY4 ztIQesJDZVzfQk4-jq7j1y-m>a|AjGf+p7S!* zGq)iXR4EZ_g#{t14IP9c4kTNTxp`D+3__G}aa0s}JSq$3C_y-yM$s#Ec}-_#2Q^iWUS9cO&UE-6h|Y4Sb%FwNl9U(zajJH>CTcQW+wWt}ro3R4 zge`G;(Yk+&^AC1M2_Namg3~uguiw21rqM=Trhc6Im`%~y)JismrVL=Hytb-JvI5Wh z+syh0wdHq-l*tC7ooPKp9K|Zg`vLw4!UQ!}(wDU51kpkt0{LTnF(F4+G)}0B&X#vt znSytc(dRsE9J{7aCyeA}>PV4RHJy2&HRuYVQ;I&l zT9j@;UvR=l;0sKX1ezS52-VutZp-8sUh1sQD)@#Br2-D=?rv{~L%h%mrwSi{2nIKs zjw+aY&13kb+P-tw^wktq37OFIjImq{h=sw|OB=lCz1Y{y5{Q#Mr%?ogFM0C$-7%z_u;?2 zUa$V&Uaz^S`^lyZiP0KgOqK)(L4r z6Mhkdv?vl5#JoF+qll%()Y8`EvK%qr_?2#c_mcF3-NT^g_sF-TAMEwzKQ~*uaxoxD zoR*)QUlmEppF?@0${T+`)VJ;JocCND+o5M)%tXPa?HzM08seh>uMb03ibIlwxv{^H0h}9}KbNHDyzto-t^pTNq_q z1fIm~KODqryl#33aN8MGsy)Pu<71Y0Wr5xedKEcj&GYX_l&jO&;`84>=s&Z+(1FC{%(5+2Az`6PmMmwaMaCo1X2{s^{71 zQQLLue~q|O80?8bFFeREcTEK`X~3;cXbs)3LA#Z}c^jlfb9PdagnT+J)3}&*^%+FP znK2rT*t5gK!~Oo@ix>Uf7cX{uVK}Cg(!Cyieh?l$8?)#AuAOV+ZQp;}KadN1<(j-a z?;_zNnkXJyBr0em6PdOm)D#A=P5|dp7&=we6Vwx|U~eVgY*jYA4RVBT)qG6O*i?B< ziz26SDiz5b+}5@n_r(?ry#u?!G)^PY6)*_;FgiQ{Ua;~wl6$$~Z9x7ZcwJF-!X5sl zt2%+Zy|I3P4jN8ac!>%vBQf$z6`}5?=2Ob%rZ73wrs%xsZ4jEyjH^g}y+WJT;Hm49 z&4f-cJEa-I{XUg*qooc8-KmuHHBp_oI2Qq!16ZLUVySVjQ{Ih^-pj7BCr)|nMC?zs z6KW2DA-qfrma~X#MI|cWS1eB`R_#^G%fT0Odq=GU`|G)Nu8>$gCgF@faWH*t5MQo0 zFH?NFMjc!q${q5f{1cBBILZhCNvz~N&UQ*xaMcfXdqHo`h$;|(_- zRK`-4gApU)gbS8dE8#;9kED>ytSUu^4=f!~Zd%Fj_ z>IZeb>UKmCzxlIqp6m~CijO#Fm$4KD;_uIl^XC<-)T#x^Y38O)T5@~S0*+TI&o7N)z^;=n5DeriCCsKB&D^0zm-LY2r^ z2xFhA0Pe*XgD&UR`kiZAt$4+DX%+|5Xu~X4PH3&7J9we}oF^)hHcGzAcxVI1ph$xs)bC%w^l!7YoX5n{Mm4e+gl$R2cJJn#*_&fvfJBcW{Sb*&)7}^c6*CQ$uRFJkI=fV z?{I}od95J>GmaX(ENpf;WlzUsP!{l+uoaG-RXy3;PvrnZh)YsDfL}l^4qPhD{Tq_Hu zwpHzF7ZA}|jveK~5vMi|;7j7?OeL;ah@m?zM^my2rdYBCXgBI0WY)_!YzF$v&m{zw zS5PiLy8PRE3_#}{I6Hf53Q?h2h-YrT*sZviadPD;@^QROQYKj-&1YD|l;!9GDYa8u z09GGC<-RrMWr~RtWc$}~ZhW3M$oIN!vJKk|M+^Ut- z)Eafw5LF?CJ5YNL<7$h3j4BDt-Pw$K<5JL5;9UFY= zp`Xv0E8`R5)H?w{2+@T`^ps$`RDPo-)vkF?uoAbkVFkSykgHzM5B7Z3ZrA=E^68YO z(SUsH)H-(n5h2fJk`G{U_xUsHgpPg^k8+yN{81@pBY0y_Ai(oX-!lq}2`q-N1(jN| zZVxt+JtnJER3p#1@&z&!M8lj5Vezlk!X~Zi>c}Zp8L1flw3;HM)M+*hyinXpPdKE! zRLWpcs&Fd23MdH^UPi{vj*R1+T}!~SoL#}-Cp_{$a35;-DVwko)J7!)Kj;=GJi>+i zs3Vt^#seEsCO@ zvkX4PE*fEM>Y}eu-tEL)2;zSLXwo4@l4RK!01J zwilBe(!!wnUb_sG_HD_;aBQ7D3w3vu#%d+zrJA*{FE~8fJ&h8nASPN(cHbITGt&XqdsfC|C+bE?)iiz8OudbWXqm{m%RV(AMPI1-hU5z z{ll;CzmM_RAg9W)!u$N$*9t%fUmkVdopW8ey+i!lTWIJQYVl=LJ9V_aS(in zRT%>N$}iY>%)(+oIzRC<1%UioTO|gh^XB)GCY>s_=76Cs+m$`W*G8y^l{3=$wjcEO z{a%LzuV_YfpltAW`&t+L5EuGcPmVKU1+B$zS)U&OD-jG@E^O4k>p$QtWpzmq?f>_d z_d~m|C+69j3|^8b4h!=WysO#<@^?+9ZEuqEab<=GoR{|}^YP{)4PZQ%b5B`*0Z)V& zCh7og$5)OUi%!Vg0oB^vfg5;sk!w6cjHwB+N=S_C)ykGd&vuEs&$Kj6CAk; zMEW74Q`@X9o#vHvKIlkx^_9@6aWp{oT}CQNRkMVdKt&DM7f1+3N5v9yUNy`ax?yvg z?d`V9pwSWAIE=qDzpdUsz4J~#^?3D-2)+@4xC#f~L>;mftH`rDE`hK*B-pMssgo`? zDB4y>s>D>|JorvHm%zySTz zig5a>udfX5K3abt6Z*tCwUB-%%ivhrL=SUzA;A4?Z&+an0cl<|&~D-BU176PA9@sQ zAxP^Wt8#4ag{!I~#+>V;>i;nZdEW>h*hPgSKL+}0(7AYQ$Km`n-3ON|0cX5GoXZ0l z>gij!L+0^FxNicBlP9IX7hG57X*cbfE`k-#qwoBV_2*$bLi1&3{wOVNUEt^2mgkPR zu$QySR+c#ryUQ4DKpdo_q4bmije%F@;3fJ!=@!=c$yvrcA|TBh8Bey}x{7=uMS*~H zEIC-pacHqB8MjUKv!6?MOH1SJi*`p|XU8i2?z3l=f!cQs?}ODwSYziZh`HO|xl=cHPgQo&Y6hZHzdZF&=2COM+?r7FcQyld1m2fzJG*tE z7?y6)IoDGv(!hzgD_Tn{%fk`K77+ZrVYXXPUqy=pc995lE?NY++o;pLJ3FnH@3O_% zLG^l89Jy_M?y0X{%x!*8LCr^K;XPAp+qu_7EzU^~x*GHVT6I0xx-(xcxi@GPzo9x< zRZ6ThT||xbhMi)Tt+_2#uw?%=rgPkw4Cx|Eo{PuMC7ay`LTAe0lFdj*J4Zd#pnAOJ zJgABez5a*Tf_C=bKHq=oJy^oPoqrbm7dlN^L}`6_J>vV%o!|dzeksDwM>K2?3|$ic zeemp{{{HvuaQAEehe!DUJsR7@g13`kfwh;QeRM(a23SsPEH%(d^-N(Fr>;%zjk4(} zPgqN0vy|Oqa3xUNsOi|YZL4G3w%M_5+qUg=Y;|ngw$pL;ikjJ_m_JAN4UF2A9G{)B zX7RXM)fN89zff1l9mveqY5XP$*Pr?C@He$)DJ8G3VfQSK`fMki;G#uj=qbVpil*m; z%JuQXfJRSf;M0kNw zI+fRm1~{{<1-$wW$}+FA#R!8O%#HeDF%nYGs(&i?(zqxowl6MjBv@|qAD1T*?cSDk z@Cx`vuMQ37uT#z%w^t0aFwxcTmA%LdlbH=p9mC^K5)0j|iq_UEL%G2U$3)i~aQEOl zFS_7N35zm-3KoOKuv|9VO266e52*#Z3oKmpgaQqGu02 z8^q(ll%*&P*-FL{<&CgI)7?v{fH*SP2w9=l-oX|*@sV^iJfsES$9@jXJAo2ontS+< zf`8A+a?5i>s9-yN)_q5Paw+T|Da1MX*8Hk|ei@22HiOZ(vTqQ{1dL?4vD5TBYmBdo z-UXwT=8POc1iLl951cntlf=l5BksbENQnhYqM4Z;VhSwojmhrn+!>*J$-UQ3E~FbBR>~C z?80DtE{?Nth#v&iFQTviMr7RuIkgkyo=_XxYWz7z#QIruIOWJwgcToXg|9{< z(DcjBG(%C^-*zsE;3*%fz@OWsd~RVU&Q7~lCzR?z=^&MvbhxBTyR7Q^Ege{HM0qI0yWP(}`n|rD`=y6ALKhSzae*3gsFj_ls~tu{=}}Ckx6XiC&;*1JI-y>H{#>AOgWdh|NfdY zK5)k+Z38F8iLjE%#|1|M8nwT|jX_@XAaU~M$=IwZz{JqU6*BE@;HP~4uBzPB!5#ux z#a+dZP|IHYEK+uve;MtQIG?1{5$F~8aPaeK)O&-+Q~P95(C1(0=JEyP_vHY5-rc%b z{9s#Xn>144H-(mG`D2T8<37Hh{{ACLFe6Nm^z|g~wu#E~^5cM(8jw2%=a_OIYIfK} zd6I#J(z|Si*ZgZWS+%W7>3SzK;G=_QHlU5UA*6lR(ct?*+6J3$w;w&p`B+~tojaiO zyUdXiA~~AzS5&(Dn2V!1lJ$E?AjLqwzA-plVn9s%((Nbf7*VEG%Px5~{rQPO=e}R}5mlj7+n^(6 zI^J-S_)EWDV{{Sp{Aoz_qd`onDq_w@!|kbftu}9s3Ph!Fga?m9M`hXaa^!Qyf0l`_ z)?)dVe(N$^yd=fmU>coNIByy#{6L62!pb=KxJb12izRcA()fU-VD9a3q!0{4p7T6M z2-=;FBVETkHV5^Ji{>^G=0d#jl1&`C(C03Bwtgqaj}!3ydY%24AGg!fVapy;&_|gM z_&!MwolY>LrI>_ELY97C zy7BW64T+!h`{oQU#ExrOCB4%d-HrCDwgRZc8@2|tbAuXL{uGsRhZ ze!|`6!26LKl9~84VJyuc-yo(k$`z6j)$hhD{{+-L@u1V z-d>yvFgz;;1JogBP?q_OR6Kh&1go2BmJ7XIpAyzsGe@)oYm~%L$6A^rHp;ear!dfq zBjM^H#rb@Hv%l*Ad@w$YzU}nMeEGlpd%1kh&%3^U z?&}5ckDS`w_TAuzX^jbH8=$$qW8L({68$r?k67k>>+5~Amq7!hJ9+uu^}PPo)5k>; z2pP48#rXz9sHa!tYi`?1y8dM4Qt`Q$Wj z|7b^N2Ri^zG~R=b`@_1M^qBv~SN;C{iv!&UFz<8Poej&V z65=8+c3N-i>eiKE^dwxyLXAjqo^y&E=-n$VxR z%ofZx&$>rmN#kS=z05BE*7)>jqsC*c63(A~`pGx(F({lDI{~qNP7TT#JogOx)D^up z$NAW7mg8;5;9JE;5`K;X>gNl+&syCRdiV6~G}S-sn)uvc0+i%LR^dP`ODe6=A?dHd ziD$7P5qIr!XU{^CXdAN0&#?eNt*n$LN4no8*6vBO`RVbeWS%yOBL0V@NYs$6$}kP> zXTfr!Wf{4f{q|Ay3|2E)LDkHHtyx5;L^lI}#hcI1A}f`+J*2qtl_xbQB2P6F6?P)}LUeaT>ylp_B4^WHKJgCy}+TBPxWwt0R%9*{jMQ zDl$x!PNaQ3t*86l4#aj0r$=ril%x`zZyT7HVJ@Plj0wIjtts7#GQ&6-2w(tTCynYE$$fZ^XKU+Ui8g-9a|Hc?1}nQ>Gg2eiJnY9IXK0Bod1Y%f zVb;s-8%vH-=waf8oPatODZg}1O4&+v?X8!B!5zDnaayL=GNBEm)8=ApK=o^Tb(YlU zW){Gjc+RtD_u1#ZW_<_v+F5dUZ|mo+sh31b8o_~uKL%)HS!|4303WWZK+C?j(yT|Op~m` zxqN~X^YcUI7Lau+kkMd@V^G3tUOoHg>XYzl`O*J|;O54c5AdI}HDxoEW?bsEgZX>~ z6U`x5RpW2A?h;H{u)L}*+J><_BICiNkg>;Dw_MJiHDWFq3uFUkha|V&a?2yzgcLnP zsqI?0B&t79zQur1w*c2tPZ&OONiq|~jCgSUyZ5Wlnmw*Z_Ay$jY~kY=Lc+N&D=^8b34(1UwhrH4|& za_Pc!WTuuV=sm|7?3`y`>BXo)Dw(Qstt=#Hp{|XoBV&u*B3W*fD}H5BfUZq}8olJZ z>o!3X1Ao&rSQ9oxpU31f*^Z9PmkL{5&Z*ZSw8EWo?>c71GU3{)dPSS$C7q*3ro% z&;>PEl2{6tkeDKI-hsq@cm)@;OtZBz0N-P0R#}CdP3)Kz9@~w@300TeQOK3pVcmTy zImEyu(j`sWjTRzL!Ec{jO3S(z1t{_&X*+3-LNQeh#*?&+9XKlXu@T8}i0uq1qiHxO zx_@LGUJF@Vgw>61T$WW>&U7<+A9x|;4=_$;vy9?edCXttM!_fdeDIzE%qLt$-89m$ z+j)pI@+PZKlRwGIXM1|U1r3t=sdRKdu>=wPMK)dFmg~@0Biji@AsQ=?jwtHF4W7`7 z6mqLb?8$+kLYi~=b8_`&Q;?c%kz6{jcGHfBp}R&0kUAzO^@LhZ!$6u#`LO9D#=o|+ zAA3l3yb)2iN3yx+V~xC489SN;I%3>}xxr=9{g_QR!gcKpqwGqJqI;gWxJTwX$oXBb zeHV~jkbuR<8JH~0hf_}5&u$U^yt@dC)Uce2I}ljaUrn+@BJ=Q`IE^((de^th(;i2c z0zRc4w#5r<#&qOd*$=cBKXttnP6xbPJjgtEGN@E5)l4*=qjQx~vef9be^1nC#Nf^G z&J+|Q-RP^Ye8z)0WBAH2X~m&2z7v|7M9ZdhgpWQ^ykp^6(l9~Xj35}iIn`AN1H4q! zXj85tm5n3Hk!TTdbmkh$EV94Lj1tprusi^_fKP*4o`J*(a)qATF-I{^)fV$o zA70kKhF;jbrmLz*G+>A(1+v|=3z@2-we<>d?hUzS;MxNrVkyV$He^&D^&{>Px_qC}KSHN1@mxiR2@pWbe>& zBmbVP-(b;)utEqP{VIe8d3JuroY71zGX`(AFDNyG#fu_^jeUt&Rl+k_Y|=|Ca7o zduKNmnHR8?uqqqk2?u<76xRJFTCO!X7V9qJ^G1Bp=l9HL!Ni6#ny`h`-YQqCLEjXI zMaC(3o+frA9;rYyKF*vYVaErPHZEU_6~V&uo);B0MC-ZmYk1UyJfwh%Rh7K5PpxC9 z7uV-<>M1wXluWeCdsZLLQ7Sg;6o={&RMU8A*c|cf=f8kkB=q8?>S9hxUmzKE9pYXbfs3X-$-@3EC{tj2s=K6Z&CbJkpM3t49ctUGrn1kTvv_1l$Qh71_d6C&6_0EQaW3!;WQq+2$aZ zUHvZ%<-wu(Q=C#o`&Ub+&7oF_Lijzyd^RIC$Cw8%?ZEn|)D;=i+PFy`=kTdX?FC8{=PMD#Lx{3Y1Bus$}f ziZYrrLduR2R%SPEl?!#*1&YEZnK#cN$5UL9?U-6z(#Fh{o60^?$@iQHzf!@Jn+Gie z&(o!lCntxpLeVfndEKewnw`)bY{V!i)*a*)9*c>< zwhWR!WXPAda&wqh>+WMG!zIO_dJ(r_NP)Re{)S0^%3^J%p<1C$Nd%rfR@{yzvO|ci z6_GDUt$;yGB@1m_&PEf<5Jw>$uh!!UKiJgxbL7b#nlfjDiB+l%zV!@yTCYKq9^f-8 z1mUO3iBX!E%7h=^QbZlUEq^FVeQ3If0D7dv`nMu`QQIJgSLx6VG{LxhCZ@FCSuY>S zX<575qB_16`;VkT{gt4w^I(;j5p(BCC1w}QyY#$ngvv8AD#D+LxhjWg5})TzSQmP+ z-{I2v%5LoUf*%;LQiG@j)D(G5NU{2?a)a6iNd8<;Ti)QqbS4;| zZVf^#Ay1_~x=j_M7A{JBaQR@xA)V1=>m)|CLXogC$8})0ZYdX`Si z?%L0tqxPQEyEKIzX)KW7dq!x*_ec8;hFi&s`kIpN^!B*4IY`%|E^hx;MBS4e1_vj} zP3sos3g0aa5>iC%hjSCkl86IVs}E=$C`V>$&zYPW+!Bq*H=FU;(u65zjVVgsP0Q$h z6Qc&5u%_}MCTU9Z@siuCPVn&2S%ep7(so0P2!hAJiiiGIp3No=rJ=8vv#LA6`jtlB zp-SGb+LS3hpoe>R=tAHIKq69WocCnPDk(ES63w=CVW2qA2T5NB%R*VxmX@ojPgDYC zwG3SvYOz)!Rb8!l3?%dVOxNov7k^}&Zkyg6$E z_F!7Q9;6<#9EC4w`sF;hH!y4@vjNlC2ycjxZB-2nw>9CTHd+$Y`KB}CzGN5sT0vc_xm&wL-r*s&*Ptci`dWsqRpjUKPK}aJ$O&99w%yp> zVg0Y8UzcGW>ST~Ud)smNR+k{VKzK<>wUcbdo*?Qts}3%M#aa6Vj=;jFDP>O}>#aP) z|GMGrsdawaoFBUk?VY1uHKC9|oOW=KxI>_uPtFN&o;LMq=lFL@GK-pIst z5y=7<+3$6O%{e;;Ogz1B9eRy#C<)wwS}eev^g7n;css~n;I7OLOG6-jz9?@G8mYL8 zQ}A5rxz|I@xh0u8)XLMzgw_e|$Vb>-5GHxkkCvVqw$P4X!bMxh!8~9qG)$EYDpQe;h8@R$ zpKuvTvwF^sYYoUCg>P13EgR(uDmBo#ub&YWV-WsX>h?+1GW{}rV&!=X83lbbH7uCL zHlo%|gDbi#MgariauH@_daW`uVi>%}2N(Fq!Dy$IHe>;`?>Y1!h#Yy!_l0B*$~mTN zm~2e{FkQCyUW-<1$FP0{F}#1PNPsZl3oX)6e@F&nNicx0&|ui%frB*5V>0E{t!;h- zdFR;@Yh(5C9ueN-4nr^9?g|Yajso2_S#rj4HE_V!G%B3C|VdBEa-b8wT+5 zRGj)>T@(6D10JaeC4Ut)ZhMA=G#5bk&FPyk-^vH{AV@+cMK-eoyJyqpu9-Y`rT-W( z0tmi7iC*$A!+J(Q$NENUG;m%Kf|Ye{&TzbIRNvo`sZE3<0hl-A&2<= zsQP?R)Hg^*hd#U>@5`T^p3+dX<`P9x-0t5-B%zhGS);EGyUihi;U$s8h`h7%h|m}< zy+omP$c#A-DA$n|X!tZx+5|8fk$=A9>peZjouG5t<|%X-pGGANC`K&UjAs!L5nn#S z8TFss4Z7E^L-#Y3rC+?B-h@0}ZKm>Gy4>nw+k64Gfc{6Eoy@biY7d*limTepF_(Xm zrQ#bd%OOPnMZL7>e%81u&9Cd8pV@Vnx5Lgi!)9e+&>13>`oFv zN#^RjX|BZ$Tl)d{k{N6AV{yb!esDl5kdAG?NNX>hco_8#NWyh8uHxN0YbnOMo_YUD zZJe#%doHKFZfvGG`5hNE^l8%8mZL)TuJQN)lM=*BWzvMlX#Vl`m8eYp^m%Xn9ki?? zleJS3ro+i!k;H5k6lN2*Tlq^tWUwFp%Y)0`Onge(bS`LmIM*KX0Yopke?4p7p}b;p1Nze zJ2D%lHB>c0G($BjLmR;s)X2cfg!?aKWIjVC?T+d3CDxaeZd3L#>lqqEXKpM^oIs<^ zko?PmZ;kARb`Q2DQBS^#QyDJMeXf}hla=k?)xo>7TB+jyQAIcWRnjZaS3p}S%e=g` ze9d)d$=q2|RoXY17qwnJq@47dI*yV~O}NhX$lb zkynCZy&sH5xF}J>HhGb zV)SzMqYIe>$Xvm6!!!BG%JxqBZ4fphOGaUxqkdxhaxY-j>RZ(x8?+H_#|P2SPd%+e z=SF)2?g+`7zaCWWkf|BYd>AdJX7*f$8+O?aaY!S9>Dn{qNoEO>w9i%yi2WxPmF<~P z{j8IEsF25+==emJs8|_*Y*ZHVSahmdr*mR|NqHRbjCm*cz=jMM@RPLQsMHYYedaAj zy!d$h&XYtzf2L0lQmU2;6Z|90_Xhe-=816l9*m#7r0QScj-v0a#>g=D$ybsebt%+R zLt)p15GQ7C|3qDYG;QDtuT<@3bC<-iI)NtCL*7O-H)_CVyyf_mq8SWvxtz(ua>6wr zy)_et{$wmz=OrYt_NdmS_tC{bC&KS$mCF+lkzmd`4mosxkXim`sa^Ak@}8>vZHh3} zm_3LC0vv>Q_KB~!!NQe99=3y3`4X=UESz?n-;M7=kApP1hIgt12?L-_Z?jN^BRZ2U zBW&wFblS1a^bDfs$dEiGNgXU!>CDxyCMcD0$Fg89>~x!>Cjw?aQ)hUGO7KS54k+#x zNRiWowkmD}s<&A_2$5({Abk7PW&Acnxc3~!O@HRe)>&jpg-s*w#)^&=&ZV%ClZ&J8 z zB1-a@UvM&+4Cw|GtkX9Sug1#FvXmM|gj?G12NynqkHF`i1ZVa#OK!Q!(9194S&U1% zaG6+fwUbbVH{aba-KWTnD38k1+H06z_~Y4avbHRks>0iCBj{XfICPU3VphkR@2Bix zq00$=Ul&jh=HE5{zO5Je^A$9c{^){pDzEj&&}EA! z{&dT{E4oAap~1~k@DXO?UWKYVz3l~XXP70 zeYDZ5RM<~(U}{N+V1&WpLWOpnkhUQZ#{yeUV#+!)35mBZgb7%*6d`xh+oYI;J4YET7k_ zmKtp;att0;nz*Gkc3u0?y^+=Tw{o}A71m`+*jOK&+ zD&(kmzv;fRO|Dd+a?~4z5`&d^&I@*hmb-^et1lZ7g6b?gHB=K-D`mgT{KQEwZTk4N zqn+79MgLD3bE5kAR2X7}i;#|n*F(yn0Ww1kZ^eLUT2XGlkB~V^y59>;C!d!BV&I^c zS-#zTxe1dP`gq3EXY%E_7B}a!o+%(Lql5YS)_UAQ%5ZgD)0k>D^Y?~D92W`l+=7bF zOweR6mCGa2PoXo^)z2x7IUedHEx#TeU+hgd3IzCE)2Sh4v1Y%L?69>mkM~JeA!Wo3 zk9rD=%h#;C!%z)fmemcl|D+oqhH(IYLCMpKhxS>xDUW|c2T1euGe=x$CWKSq)~23K z)(H01O;X~^k3gatc;9s8x+X4$;!U#+Mf7_TeRmHj>Ju;BoXbhnn_6xS#8TrfOE*oYy z#hyHL2|Jy9?~B{K*?ROD?ai4U3M)r>ueoK_#eRAn#MYasbeS7?ZR__IxBsVzna;7Y zmOX7}#$FIh zv6dI?qZSMb#%?-q{eQWm?Tf6G4hI&>9P}aP;VsBtGVf7Q}!hDu02%GSa+Q4u%DC=nuPmO^+` z>foY=aIWKCoLTU$HQVN|m*8-|Y!tW#cHX8N{{$Xiw(U>)k5blG#mdJSY%)Hu@SzC6rqN0avlxLkdeZdqNtH z|9@GS;~4SVMN_v4u8_up$@Fdasr?#=$MOH0hf&cg=6J^Bu5I}QA>q?{Dt?y^*IG1o z0x6v|dbzSbQiu_g=Tpw!!8uHH!HH;UYGbEvo)d}^@OyPwIKZBVX&cGomYh_TKb^#Q z1N8npCd3t(b__Uw_Wt!{4_Mz>egB!-ys^9!PO}i3U2^qgjYk{<{me1vU+actzA8v@ zw~Frq(k&*d0e@kcLNY`~RJXr?$sohR!;xjEhok0&gpN@QvBTs?Zu3P2FCy^!U1 zc3dmNIZH;$aTeC8-a1%G^0P11iDkNwT8XAg)YbK9=JlMPw?&18&V4>(I|$oYu~;& zGDP{03?E5c72&9(XXLM}>f@%A4v9uIa8W0NxiWpR1nXG|a62XMD_+8_3Pwln+Qw{Q z#8?nj(fit=2?F8u{jp~fH$03SyS_HGYYNiW^sog>d>eevS^B6hv$Hotg4MOxbth7{ zm4Ae#%-E*v75L_pHsD`Z{&m?(+ZR|IWpFb7Lt`k*Zvl_#k3Va$b{Zr0h5xkgLEXK$ zD&E)2l~N`hck}($^uZdhKv60g>qFAz=2d2IDTpgmC|iwsNy|bv(+UhK9I`NTTY;@L zZM9mmFG>*ns5A1<2r+A23-L?9^I5xK9xP0aQ`9⁢)x=2%urLwiLx6Cq;R2E86K-1VS2QX0Crr@sq z5GApRRYm#H>tQ&mkF_Ub(Y!(PQcL)XeOlcImIM6g4Ep~o3DZC~*S$L)Op?V@|5kT( zCgFG!0CimHprBK6k1#sDGV8w4$=vt2%POlttNrW^`|@6htW`O-jP$>SVVJYRlR0}F z8hiRNQluP(L5-XFRIva?)YNWx+w#U zY`pL-9;g{4`&`FxdQajjCL8H-8zQs76&KVwaYg?{nJ1hNr0E9hGAUfJjtAy43gG2_ z-<<@@G#RZdRL%!)ZLw)!$(Qkxe0+SsZq9;ltG0-YSzAE%PrF|`tadGJG6sA3J-)q` z!N+j8VLJLXqg_tN(f6bo`=t`plBhp#wM5~&9(vXq7aeuR7Hzg_UZPK?Z%+56gSL!` zphGlsJET?IKxa=}E<7p+nKahV9CYoIG(Y8 znV2bI?Y3xxHA8wREfw&tbs77&Ezh;q>0TeEc1w2Gaa8cx&}SRzjrTlMwR7*iOU-TX zz?HqT%=*(3i3(U(JqeUlG1R=Xy*@wg?m3^n7??)WdH@)n0o$9qyY(M{8@@|lUx07S z?DmfTHV<4=LLhq|)x{I@mOrl0)#MIRwc~@o@9AzD)i>ku)BB$HeRx+lJCPq${3-_9 zCp4bEPJx$&{Q&vqo4Hfo=fj%jy6eK%14D1}h^4|g#iTD4;ZW^d77W2qt;3ANizBTK z)DS?<^zdWczgi7Umjb}&tzHjk>jS8Ld*{qzCIXs24E`41?rZ)7^|+wlUpI{_CMfe{ zq4Tw^?|56}4_Ms#9H{1Z9?~&B(7s#Q{9gFzXKE^h*U096I;C##`ZZ{>G9dmD8){w6J7^@|@xH zcy+t<5vNSi4cw-a8hwRq$mPN&J$)e5=}Pwb?)KrO95;!^d&eaVl;mniRSp;yBeWr5 z8NS+r@Kf<|RdQZI(~SjgTn6KUbVMXzFOri`kd0j!kXblS(C%fC;vEfgIe3bq_>+vN zy6=`Dq8sZqt0_K{-w-k@UpSRO%r+9GUbR%}CoB99l2H9i7VpJ*@Q)k$RQPcWis?&YLaiuiNwp!5lPsjUVza^c3ywR>+z;<(F0E}+h^9mEa|HQ< z_|ww9@Sng*c?a>io=FC6G>@)xA-8ZvDOb|JVt&X1o+MI)zy-)qCdPOZXb& z^&DdWjo|79QT?JB4{iyaO#>RShc^VzLH=P774~XoS|r;)yK<~6>dC|Uiw2Hvoutml>MZ`QlqvJqkn~7Zxd5m?&bLrDxc>OAZz-L z&`t@s#s8P}=?}R4m@W4ChWhTC>G}?<4wybZ9f(&=kbmCp+w`~}eQIJl$QXe>_sL)) zf6GEKL^cPZ$eyFEf#1an`^s96?3!a+>cMHj zsGF?xtSTgIrD=<4q2);1BwK72sd}eWglWkK@4XOtZ?Vjjh4@!6Tah%yp2TJ}UyKjT zllEPIonEelZ$LQf%~%X%H4IpZWMx?-Ku0?}%x$8PsgVyHlicg!TqI#6LlA&CiSoBr zxYkYzUUy_?p7b(+bn1Bv5ZB8~(Lu?9t-2}bE^L`a#k<&5e5c^(O+ zPP3&kgv@(sT2h6UQ+ksX3CoSy0@IMvRn&vh{fGZZw!f-Ll3SXz9Tj+*Y~UfKypmN9 zCRo&I+D6t4t!kDkyc1C=7hrtYO(%-S49^XAV)JK@q`}ciL_K7FE(TwQC3}w7m^zv* z(Uvg`QWA9p!lGn>LC7ed1z~gyWd47wP}&O;douP);A*B#Ro%xA%S#sqyMqS|l6a~$ zHUz+?!MVyVI)E>=VjhIIP>X<9RzM%ol>I!LfL5fORzd!o4iYx3s)VOBU2{GOy1@<6 zwef5}>39P5@8lqK`{0UTa=jH%V^)t8lRG8(Vwa_2SKWx5#M0oQU zBrQ_IaVi>wrBhsC@miOsogor5?1zN-swINjb8asMic3BT+i& zcXV;W^VmrwQms-lQ@%XG_^h|r3L8tS4OO7qU4u8JG%v7kheX%)V70!M~inTY#mJx*$zDKj%n zOux}+@2d~+=>J1t7Rv?hgW&V{pL#!cw+)t?&~DY zm62O|`RRF3%S~nL$+(a4X|aU|p%{?D3F)-$_m&F@lCc@#|NZWsidEhgFV&`na5(Y{ zK8WcyqFR>XTS~C`Lz^ZYGXY)3MsJxkC7HpLB^@|G=cKcsq-%(M8)6CmEy0x`Oo}o= zb(&5WI>b9Qy;Qi5ZpJBe6gP=ibvB6QVFqF!L`}9CHt8h;oI~S;ce9k%Afky3u^Bj& zGLVXr57uD_jx2_)kJAM#*eb#j!gW*`V0DE0yFHdS4U5kUeoKxNbfmUMlRPHr#dbtR z%C8>>W8P_C{-QfrQwfqt3hlZ%0rPi+Xc`t~(itTi_U2;!F{3Jy36j^;M-d-bLR)Ce^Y8InPS^N|Pr zkR;QZ1Ui*%3s-23S`m<-hw2^K8g>LAPVhC3BZK#r!7gG-FAygYeb%4U0a#8l?!yUc z?r&&Pjve^>wWh?B_P278Ns%M`b9!X8km4n`s_Ie#D-<){Ne$1{~C z1y74H(Zto6Hw_I+{gKov8sT9oQc%&9Vrt2^`Z3Cf-CZr~qPu!hBGUB@pEvUm`7f(H z>w+mn!$>8}e7*t>)3tLr^s3$JDiK(vFKyd(s|vf9Wt>6Ihr7w9cc6Dk_mn#&cWLsj zv)&UU=s+=)1BU4XLmXRGKG@Bt^w5ha((ub`wS%hadPUC3w94`5@e76{$tsCNtu_$g zaHU__F!DzbX1pl>WO4UJJRJmaNX|kZ1Qyg&$>!}rs4+vrxIPMX`;X%A_B&2YE+xxC zz{iJ6S5UInj+mdG~sD;BI5JsdBh73{;gg##+oPc zq|`8RRcgm+xpbFZkT7(t zS?4Nh8tW<9fxhW6RpfAqoyLfUrg_C#*0hE4y0r47zG=#k$L)I&9Z~LKX_)ZzOEB?cCfvCj7h5^CPNC+qERrnp zXQ5l#WXN0e+0?oV)-!We1)BAW;t;&)BCd=0hACF)4y%kgefpKUAP;F8LtZ5*#uD-%!YS#Bfwfq~uhe-H(8bP6_p{p@M zhtIUj^{;5oXvLg~<(DO7IfqDus|SBY-z{G<%(b9p;?wv_udfe*UhB}2K`VHaWO&R7 zvw2OkL0e-h>|Pf@%AU(cVZ`|c{%o?R6dVtejd*`-=d>6#c17a4nT|}Eu=p9(zL5N;-1Twm8g-Ig1KhGk4o=#Ul(zbN+=Qby@PcNfl*4g=D! zkE-v_#o}oz`#UzUAr_Wuv*u(wO@BE~Q(E`u3@z z06*?`-5m}VC^{1Z7J8NxpM4x!RdY9cWk;6Dl4jbGaw?F@^lut)ciaB>MbEV_kpR_k z@zVi|q%1!nB-_^cj*Jq27+rN#3u!}Z>R*3)#l^v=O3;;2SL@_r1=k&nL7c*@EQdxM zY0TU<+vTdpv19LQ@el(2HWjrl2Lt=*0 zThfkAum^D2RMkQ-InX|;;U~kqXXVP?o3ViO`S$ong#rnv(9LKTEHO3vl3Z^X26t}c zt?v_f#>JPPL6`TKnwYd!l0T$-H;LaPXR*TlZYTa5-?x&mT7!F{orrfA=+5I-Q48w> z>L4lI%CH!546SLY`*$8X-YU#{2plR`C2IlJbn^yn^O3!~)F$&!Y5F4a4(QdS`t zV5QP8d|YjyM&j_y-0`M{mnre&QvFBvm?D|vfkWFC9VL01knvM`6zS^Fb#QsJZ_#}t z#j^!#GyW8zH8bI*JcNORy!w4o76|pz7d#kydpc( zf~>JNQR`tdJZ2k>2U*_DuVj=*s2MUkNggzr<^sLUk;TfM8e}!5qI86|qfOK)n?T?F zV5Fg=ufH>3GV$N#m@#%Dot}Z2Gd5x+Brr4o_EfCJEgrf@ z-6WO8zfLL%w0J1%6sUXj1^S6%zNPBI37Dhg*_o#y)YC_Nl;+@!LU?xN#;W12AYucJ zTV}<x5-A6N?V-o`Lr-dp8%5cZ!gBecfq1R}ofaRD(PrRCvuP`%P} z_W6I&-2rA9GgZf&^T_D@A&azje8Ka0Xw7_^^UdxBW)Uvm`rKb$-VMG0gomJ&;;OAY z@d*hSZ2OoS)^sP^&<(4==YlMWD-?Tn&ZVf{Hvg+YQi8_pn_6PYr`CgrM;o36Wa&RQ z`FF(sIYm>U!Y7)(7qB3q zD>~76Paz=xD}V&Em-atQ48QQ-9~1Lm38b0slQ;Ley=6Gq(|*zK0oW}@iQAoblRO1WVsLuur7?(ltqe({i>MQ3{w)rAqdj8FpmEUI zrbUq;v06sSLhB(|${bN|5*)E`n4q-qK{ewVz@asPexokPscrN43hdYYe0t>np9a!h zcbGdx=ON91@<-m*6fC)X%9V-idcTOrI?%%8)i8c6ipzyom$h$C?C2{0k83-GwItdZ z;d4O>_;7qi%6L$6Mb5aV)MsM8V~=25%}%$Bf`w3U^QCg1>)671u0T3gtvvXDRD?Ey zmtTr~PEO6Lc8WX@7({L~S8#c990nF9hRvbO1^K^WoY!md6XH^s+&=K_MbaOF$g8Ht zHpZO<-+Q)$^`3(}^<8<2B6dI5ITikepJeMaO_Pw;a23Kc8gj1mn`<~asVm8gVcB|@ zJqNf<^-XIU@;a?mU2eh^UqaDZqdw4NZ2Gpo3?7@q`zvFK1yW>*+j5L%_*uSG&D3Tf zD;4S$)QNoaETpf??Wf<{41}AP5idNJ{6Q~|S(12&5E!%0y%g(JCne_IteGH0^!(6*+ z=m$H#S%)O{cUK?%7V_SIt))2z9Y2Mh73s_KpGet11cUyFQ4{5|31U+9EB#$1EApRt zQEjH-^GZ?~N7X(?9AcHJY-Ry*_MyAg^kM>&gNT5B99HH7RIcoy~$y{IQ!6 z9rSRu?R`D4>0z9~s?nm!i3_yO5|F#{j6w1*3Y;^e;xSq*4 zX;6{rq}lgmPj%Wc!|fWw7rEN%ljDb^6p>UB7zh;^Ri>@yjL$CR+G;lCTW-u4Ekn0t zqwX(yzlz2HQBzaX-=?Oz%-S8}<4cRDDV(Oi9 zB50$*Yz9#TlIn&EP)!%?mmqz)O*FA<_N@0i>{T%TZEeg1w6vN3%i82btF>&gF6%sd zNVwpXtvX6JHhD!^j$}5)!3$^cW-Nskio{{Zw&TIknUK!~CyVCbJVt+m)j1Yx;iM@? zH1U3wPzn{78xSx){!13_xT-uD<+{@rrG0K6xOEn^JBsMY;a-rSm6sb^ZYx3L4rC~t ziw!&cu{OCNOSsgG!YW7o+B@pcAQkGjJY<*x$0BV6pqhIrC-fLyVa>8W(IIP& z2O!sVg?tsgr_|}Z+aFSz6+T1$Seqr1?+HK23XV#3k?#Y~cm9f**-dW}Xl2!MT`2Kb z8f=hZ7`0RPv*aVKm`hq7it)PMDt8nD?>bD(txrB(fw;>+9d*=lE%4EzCXRPB#fTGk zJ&~gddi?Ia=~NamMf-6EsHgp6iDsU-3l!@)!yaxuaq(=_m?2v^O{q~(rw;s|I$fPj zo5@b~Aj?s(!P^?7aEKN^rcXeh*485%1_uL%YqE#tobe9+%&QL+J47T%hWUG5Z$rhU zXil2(@1a`fB_uos@{hHD6>ZxwE{zWjW1JjM8moiv+Npw~eW98a({Zm_MijEz1^A~d z13w|chMY=W1P3%l6pD2^Wp81J+aNt5F#7=p;8!IKSB0jV_=C>`ZX@ancyn?Ovq=mI ztq{;R<}Ty(nzeTr{h!AML%j)N87y&N!lDwlWg$k4W|Aq)%d^t;a~_nLY{$ThsIe+^ zP{B31_pt3(VuH4c&~8`O@F-ahS<7?ohEfIai_6l9O&uph%!%+K9}!hppfOBKLzttC zh!@chj94_T>LNyy8w+`uVN4vXRU;<9fTtl%Hy`)DTaPyClxg;#d(uu|?HRl>E#hh? zp!T=FKCW-u_=MGs^&7Q7KZNT^9Uo?tlc&HL3tMX)#SmH{CRxN1ak$dFwuc%Ta27@( zB@%WY?d3k|rNDgIjihE^2vXiyRPIndZWnuUt-e>yH+g0WuKHf?FDw2_+azEH{Dbj= z7c1Bj6^Uh+`O2hme8j(f;{YjmnA-&W<338~jBTWqmuLF*e=v7X;c>PN`?q5|jg!V^ z(^!q!*tTukw%ORWZ8WxR8}pyu&%6Guek<4dY}aU;^EdlZh5h?Fyd=j0u|RK?B_e9> z+r^(f6fnwfN%GA+vqqx{7m|F<1mecHxn$@qw{m5C?i~8Fbn-`GQ9tqWp9?AfHo^i? z`g(ZQgIfpdj!QC>UHFFMswJ&(L9 zq#-x(+uyBlYDvZ$t|BO~x{JlFEY&c7Bq0R zA&!QJs!A;~TuDYIx@eKd;+c(JlE>DVl7W!A1Z_6F z-%wt+nK*a?21ZlR6^OAUt!8+*)eZ5ZfpqpC+@+4y4J$4L3qk|D$lLcz>&1_<^cLU& zOvSa^4FU3tF~oP4awaXKr|@_cE&|-iGwoiPg^x>7e^=z%_WK}Toy+CWbEnk3S2UNs zH6lR>O5B?O_$B7m76yf;N@y3dMcbjCDzQox+jX?LGWGJRmlq4>aV31XwjLIeSU2)r zlhXZ#AnX7IE(0}(hmb~fP?j>T65;nzx!CP~oVj7r-A-5p>F-k<8g|y{CF)Jjq!U&J|AjV@C#6)pCbh%8w2n7upE*kK z`yWP#1=gsIz|iK~ztAQO4D@!&fr++6XFySMSX!I6{k;487~XoOxPs`!Dee4}=tP&E z#fme;`RCg=x<0;J;~Wj&q8hoXlIABUqNCjGSt=UrV^h|EMs?QrrB7m8V&NN;xiyS$TPgS}s8*aDtMYTtzQzjR`uLPkj)JDzs9JnOD zh00L8dZI!-u^sPKmIge-v05c~z;9>jR4A9u(+v$=_3wg2c(pt0y>w-H7x~&0_`u3W z--i9}+_Yrd?S%m^3bcH^M%|LFZSvBQJ7o`(4107wetF*P!6}4p6PYu$fi<;J4Sc>Zg zrmy!nS=lEj?+J>}yr+^<1J+S`jsv43lLuq_Uu9#EXj)Wgo%)yOODcZKfscsryWwr& z2%)ws@-2@-b#+_iEJQSe`i}!}yW-Cqx8d}A@-#lCEmOoQQ)SFN^G4wTM{sA@D4r>W zL!dfytNBAl_fY+@s?Ymor1IfcGY)bFa~m};?%niakC{RLrp+_v#O;hd?$GH=X7xCo z{)iQshl-_j9~F<_nWu$y)++*7$HAG9-<1W3)aF_e(ct)&9J$z(-aL!9M*_{shDE@V z$XT-@4b);ify%lU{Wnxs$aJ*)A8!4uN0rhIkQ&prf}(;nIMSng`p}jPQENBep5PeHvKVFz!uo=o_wo0&$bY>Xg0J%mVP1+1X6hBY_`M&#%A@kvVz$ zlv+@CFS?wolKYWGI+&*C7M?35ej#HQ<6J4e>OmuM71c;!WwUm?e75tGWxIk-49vL= zT@H6nc$|<;duz%Kw$VLGwogno_`zw6MF!ePGxYBFX$k16=abcQy4YygK8~Z<4xDH~ z?wpZNmNCkB0E4c{){oEzh$ZsaAN{A(M~c%TsXQjkmJf!H{6$Te$V79ABEQDPXeKjK z5=p7Fks3@Z$!v%UDn-DQ%?{%&G&v`SS4cTHHjL4Qk5E+7ueA%AD&5 ze)yU`_dx!a*{ojKs$The&a}7^Uim)XzOz05$~e{Q;(J zd{Sv05Nhl@^L++GAoj4WlXX`~s>DXoZHGHmbfKSHokd0zlMx~@-g8^pz0D7R<1soh z=5Y6%|8qqwE+Pv%mqB&g@q@T$d8CRg#evw0C{3n{?w5UjhsJg0TSbTrtx+Ie3|^vs zbjd7e3tI~2hIMV^^@z-br9$u}(YbJB%+HUBHuurn5HFeIP7FIh@GsnVArPw#1(L2RWQ>H?_ymB5YhUj7*O$v;@E-jp)d<3s zP58V!=TuR7;+-!iul6<=<6A`WLT!7p&dSJ~a$j!PaK)UJpjV`HG#k7CpFjQP@^8{0 zv`W_4P1$FK5noXaHu*XXxaWer;^4ek&9-a~>uzP3+CHeqn^I1Ovcwr&%TUb|4N%~By&|3 zN2124j=Qj;)F%}&2CC<_pZqj;nDr)KWkp9UuyA&f`dttO8Pmi)uP!~Y>Iul7nfpQty@e_7^BS= zw{dxTSPooi1X{xpQzFrlpC*hV7|X9>hpA*p+ZsWSzn=zRO@;A;S?w*?ku3+oGt6Tw zhX7YjPKyhB%pVJGwM#O`iBg{pVx6mQ7t9uxQWw?7+$7oob`*%*pMqb9x(7U+#dT57 z5G;){Goci@dkGJC0xBnqNbyO+{?fR^sE7_hTZ7tZ1HZm;#GJ!>x~Fwi09CaTV5-PissT=T(Mx|~`+ zmA3p=GsQ6Hxj!kD@0$t#s^q-)pp%$J&Js#ag^#sBMU*h;aRQyy!p53bZORK>K?Audm+ z{#4i+erfs>jaO3715lpZYs<}VcAsT$I=ViwUJW~*uN45>n>kSFF`DfpuY>_cDV|@Q8UWkt)4&hcUp=-Y0P5$S46*Iax(n``1-kTyPI8)R$mT5*scV9c-bJZV_Kqah#tz%(77gn+k+kaFx8{&zje>7)V$NEQ zRktm5T%_tLZdI$%OV8>P&c!wB?v?j$6{LF}6HWgl%qhAoSZypdExDXG zbf?ziZ~$&Gq>%Nr;r-ZiMJ-;W5ibSntQbY5Z9Sv>!I-W{?Ene*NFGAK^KZ}kQ%*)F z?ofW(U{0T!92^_E%?{X9`vS+5kXlN1-s;dtZ88H*9rgXZ?6UZ{zTY%Oe$Bl)T#gT2 zXeGmgl^PRGFve=gdWoD2T%a5^&R_F)<~N-@pF zJw_1+4fJNNFb%maCDjN5}I>>i47N-u_0YqN4yXkIp!bR2fzkGZM9QMe<~Px!q5LD zHqH(%3MP>ETn8ZI#c@5~v*M^2TCt+`>!AT64_$gMvH8#dWn_x|oR`?QMBQRua5HhC zzS~~nbx*XLiVaURgwx<>GOCijZFSVxE=s_z^Qf*ls_9Retxm23oj9!38aleJe}#?y zJ<0nbJcE0hNagbs`!ie*>oeeX_p8D+w2wujBzEZC*R-amLva=GdF+XN4CwmcYTDb% zX{oUYD8G+(s649q?khJG_sfPqtUqrVZ{*O3E$BS6oVXI#+ThZ1T2jWOO*0kuO#Xas z{6tc4ew0Cher+Lw(Ukg+Ot>A-7Wj;3U_KQkVL7;FSf$p3ai5wk&*@RX?&G)Pb7cYR zfc9#=MkD79y+&ebXO_{RAxMrB-XpSP^1dOWSq4OBi3y2lB+yk#LfmTS=z-pKN5!lYP~#8ER2)CPmIXh&^2P1Rl`o@-Nk zAU88KVT#NAAwD5(mo!MO9Nat9Qa~qB4}5}7EA-jC_MRv782?PDANC^mLsKU27!u6n zS5_fDKuphqpkm9JlKd|A`wx;oCmtO(G=8NMjB@?S zJ^26zH$`T$N-u`!v_m~AG+mtW=9oM0>%?pEakd7$s+I7VD;&kMlabo@#oxAM%~e`l zgq6`0QoX77TR?T@CQ_QjIK20si{P%IEG&2Rq2~gcd**BLUPe+QY0o)pNyj!GSxjAc zAvpwp>;^0_DN~(dTyP7%fyKyu%yWFZBGpu0^>a2Et46w$1rbL|qOW+BJkl;4CPB>& z_~rr2W1EXkQ!U?6xIrSDGb%M_HTyS*$DgE-|4F{@MYy7653WZ{@`RvG#3T!&l-MxW z0W+o5!a*gf-XMaf6rMGi3`$uCg6^gBvD^dcOrTCsK--m}y0WfS*nvz;r&fv>hr)5Q zmd@mY=LT3}0&1Qq?+(V^D~$pvWjlYxvriw&FXdp|-D(B)JutV9RZ~$W-P1+s8}F&d zPY;PKyu7hY{?1B3?fQ1`&9qVv29}hAAL5r9%T{lN0WZnDRdE^nj7e~}5_^?`GcA0ZB9( z3}&XQhffhS_-)Z@Sr0Z3p3Ktj*hFB1EaTMu#wMR$ z=aCw^17E5&qff=L$ZmA{8Z7_pi1|->f-c>?9f{dN6zcBM3X= zR^_!sC%6o(Z49!N(pP-QWTV&Zew6z?uq8(^83ou2YmCivp@Y?xsR3&n^DGXBj@Jh6VN~?f@2Ie4rv% z``g80kUwd4_Bn|gt!76;^;`6<0*rwCkyc9Km%tI)`}_`cMaeH52~oYqhWC#lPNXuG zdoAK3T|WxZ?_MT>I*E?kzfudoBvc4A#LrY6##Kn28)}v~dr>oWFBZ95V;@2!*e;~Y z^su^A)j0xtN&M|&kL;PExvGWq=Rb{iore+z79S+-7rSfZRwhEq!UM-Pv`4#Jzl(Tm zpg}^ATCyeMprR+4qOyBxz!S~|g3qD!sZs&<*BnyG-lTVc_{<+;zC!r4w0Q50<<(J+ z1POfsJA3={a1t`O1AIH!Qez-KgRCrylG9+9eo#(%D~euW7ff)w`_p4YT*2%01WD5q zvuyKEm(;=m{4gT3PRdO0>Im+cp$rnMK6BU}LY#M6LkvbRN{mS_3~-nafv$V*A3hV7 z)zYexJ#d(u97Lhc6#`F?WksZB4QozoB*-7tMjpF|8-#4nIi?76%19|i3yO3gDIyYf zQekQdyiu>mj0q;J2N%5OR0{Y6km3`241{y|F~L?wS9vD->Ua{v#N+X+?<3kBE&+o6 z(faMG=qgN>lzUnS4f<&J63VG_7?#Z6hyG}g`=eCVYU;f;6h;l{8!QMWL}*=futX+> zk#1|2gQBECf=le7nBXa|EgyUgU2R34aTqZO|G>ONKOf~N55{z1>%(EDi=@3$)+~NN%;Hzd-Cimr{)lhY?@x2 zQPSVKGz~2h4owQhR<#;Pp{f9*E-e}yJKdZKN7TV7$EtG_f>NoZz~IC}xkj$v@qy>Z}kN)*KHhg|&S$K`v~dujm|MGpR>=LkkSo+%Ym>dOPYccrEJ2V>@qj4C;$@(p0(^@KT(Pa~$Efl}%=$@P4!R8}&kLg$!M5#`e?$#%$ zBh2>QNq) z99irAmU$RTy{#ZLEa=ajnAA#JcNv8y>?5U1bm4y_L5G3O4TNhEu(@G7^T8l9uAgPY zcUG9U4a*;6Ta%P{>&Pq0(9lx8%heBpW~nsP+8D#@3>1E^H*C zJqOWPQBp+I+fjLwPfkF4kI4CbXUq)I0~VP$gtNo;vKA+1hl11TsXwK z#r^>_)QqcY71lK2A441e02)jnKm&X^hQ?y-EuM$0YLLNu?`)=gw=j%i1F@6{Uus}6 z5CPM`)wFj?62TbaodXM+rshZoU2pDKvC6y&sW;Y0<;z6EN#hgvMaxugmm}9LV$gNh zD~5`U{oatVG!b0eUtF%ZXu`C9IUs*Hv|#7d_G46>s>v1BLBJb4ftGcOgeha9*uln}Z~~X%OMv63qJ8LEvO>+o5%j+{#*bS^n!! zNH05?83Q6ioM5Y?kAkFw^Keb58R6lYf&9$A5qPXu+9K4%1UFOh z?|qg%UL<}t|CKo??vi<7vG9Ap&S^L;ggd^upw*P~hXgXJ_kps(w7sp}N~%GJa!)nB zqXRSnl652ePRK%xbX746G6;J+t=cv=%HQS;)10%kZm=Y+^kQX5`#~6x1NEWNNX!m5 zs8UUbmMLPxvZCp$yHZTw%@X~GyY+y*joQE7#)Jq32H4vO?9zq;dmHu^vagq}=GH@Z zv;r_PS`rlqhj_QG#CZ)!?dB%MuQ;FDbu56_#*6tdG{oJRy6s`Yw;IS_5I}o|lZv&a zVa}Jx%97Xh2cFh_sDjAm&;~)1T$p4(TE_9)%w-;-m1n$tD^A|#Hxhgj%Kp9|=Zw+z zC}HD@eGM{pM3wpHhO#74o(r_coOnB^LG3cvb31M5RxLmG58t=BoNZf&F|Pq%4PW!X z!_LdE>4wlxU&5Z4C$s_g5Yp~;WTyi{HQ-?@;L6%H2}4l>PCyEVHNTG*lR_}9y-*}7 zyz=&?hf?I#=>FFu^?)i_O(dk@SMH)aYO1iR{zs?QzoDsWt_nCQEbYH*jzWjCH znenf*k^HVn!R%={?U>A~phGI%ZXp=wC;>+>L6-u)_r6%UnQYrN{khH88MFK<%8pXw z7|9K@}qKO<<1E|vXzZ*5EXtw>yuZpT7N_oJUwC>*h-hX7u>Z{;KBERok)SPYj zTYZgpxeendiDbfr)?7BXlY_ID7U&Z!6_B4n3h4myGe`g))eWPGmw)^W{YetEDgMUP z!wOwYerSUKAp}-i;y-?7?W!>vJ!An7%W2fviqdFg$AS;a?>xJ8-Qr(p(_AE_bzldd z?d@Hn$u6*kH8Y=8X#T0eS0jn8TW2kCyd0H!6g@lVLF2y@x!HvUayu0^sc;&HD&%v78;AJ_Dfjr+(#hHyc(e;5~Uj`=6ioTvPCjTHQ+ zhB6MMrI<0!vYv3KUgv>k>sgJ#w9%-lR)SLsnH)Xjb9_=5my|c5sJgvKBupV>L9NTT z9ov|}<26l;zVVqphJZ7J;6LYeW({1I4(EO0c?4F`UaF|5v*1t#$(dQddA()Ag^GoA zj3<9gmg#AkK}1*+_*DCS)==o%|6t)2w&xMQFFNrzf_w^Ms<3HbxLN0{m<#)4L%+5z zDA&vGMe3DSe@ZOE5IT?GD;sE2Q=Z^e>{5<(6)w^DAP6Pnt|&<}$>Ka;Au1dTl+ecJ zI!K!ho9OFhRsrMXkQglc(Q_iZgUg;#q)#2pF)=UefyUzTc6iHd`-?hhB8SE&3%o|8 zQ6U!ZgAe*ZG$w7LjMTevZy`v_Tk0(j{R9ne(~KjgVttO9pFH7?ghd3Y^3YnD(a++s zTgc&7k{Lbt_zGxIOlKx0;fQN==9UN{l6lDp&jHzQ_Y< z3Iz&|%g?uMD&4}?oY2*(Z5jgKh=5xjp&2y)WCZNpcBvJ{a@o`NA^@9@0;Yy92r`*0 z{a(#QXpl#u1?vH0PRVuP*+*v=Wdf0i^LVh{LTLiAT&0||Pbi;6&QNQ2yv{M?JLPvIlmWd_ZuO++`i#1Q*-73yQHV?EZV#3^Z! zBcmLJe4XhlYm{MyA2h>)*Dk_xF--akA=m5~&?m)ppeEAC4?)C0$@LSaNSY`io*7m! zgl{k}(EJFE<)2y3zo@2>%g|E-*6dUQRGY2rlSDY;2pec!#v-a}fc%WBMk{dEDFHYY zTZ+9pWu%9H#AEK@Qto~_M9l~}AEHH+5#Zm>D(_bWo^xw5MKwwx&v6`9 zR_k^ChlK}@AZ7`V6I6dZ2Y`t@QBOcc61xB!lUbYiJB8!WRfHQ4HC{V<0TWbi5Bn{`tBRu*qqwk(YRUqS5cWe4%f3rJb_l^#ECfR%DkxyhScW`<;}1MqFG zmmT_Uu6mXNoEUpz#vS(svXR_D&1&+NfgBl;e4+|J6DnBY>R3r4A=`@Yj45!(q0;22M!ONY>nRUa6x`zsaSN-*2w zAt#?DhN5Fmp&7@Fax@To$a;WDXZBhTk#zv^8OG)2%&68g|07?b9t`9av%!7`{*UL? zaAAHP6^(GvTV&tN*+JK`#ek5qF7v4$8hFU{<_rgpfKZ>mzH6ILw8>bN5&C_)dkX1~ z5Lv{u<~tVFQ8UPSMR447Rtr4mT%U2gg^;Y1-@?S!tS%QyY9%>N)}#VcJ<{1$$=1z_ z1RzolKjpbMt@1%*O!Kd)X@~VlI%)P2R>SaK=797Jk~t*RhzWT0>?(zgB5x4`HWlrBD zF1x}6H^987!KM}p(M_*JRD?ZHN^YF{C(oQkhOiK+%galhAjuUs)+eWpLJ=x3E-m1c z>vv^NO%)I1JrOt8VfSorsZ8Z&xSw;IcfQ3yZ{Paqh6G-d^wcpWD;ha9%oVD@A>bxS z7m!{Vl-qr8L~pKU>JU_aJej|{b?vN>u%&44?)?o0Q$d#P)75rYFIKow-k0a+(?gO| zzF?|Kv(Zud*(G~{Zn>=EHPgW|vs{Y4y!011$y7%8Z<6UsTfg4edF4GXMH9&d!(h?k zek)BS*&%ysM9k2Bx;<_M(q-4~;GIZUigkt|iODK|HzA;az3@b@jYnHqo++^nBRD*96u?bTt^hOyUQQz_3@ zKgo8+VAu#+7ZwvSX9#79(8&zVID0M8#566t#I%MfVEOtgIV+#U7(|7FvP}ow;Bdyu zQ)INUuvpM>bspbY*A-4?+?dtPylK_A0bd-VEmpf#Bin~|fv$U0_Ne*>x9DMEX$xZT zwTcs+A7Jm}a%5W#QOpAA<-IrYY--<#i>6|t?duFYTYFFV%Y)T0e^b3>{caU;xXqj^ zWhy1HKNZ+aT=Ua#+O;M9C$!vs<*s`zZV;=F zj8~(A-Sy$C@)60FGVq7ghg*~0&$Hb!{!^hTk5*!MLAZbFjP%crA1Me@O*b=2sSf6P zBbn1xMe&Ek5cQz&-H4M3aN`VX#{IKxW~fIlq&+?^Z_%OG^?lfR7Cf|qHn_Q6=e9s} z26aYK6QDxC%0an#YPP{*H!&Po_36@fuvE%gtZS(gE;BDuQMdM&@%S}kUSX!JS=2QJ zFHxs`G(q$9M7Q?UnyHdZiL~1xrkyqq4rtCCBnU+V7>e`KKiQw3w>^+lX#E+#s4Y*I zMNVzyj9dxcb%!v5U5WSS?t1@{-)_(3{Hj<3aH@W@e(a##Fx5WeH!{;z>qHI;icgGia1AXe33PkH6w49oHoP@WV(^? z`wWL;H^hJH%%8`~@qg-!nVU@>gMWO9dU>ao51$Y1@M#54MK^0J@>g2&*9hCbfFcEZ zk(kg{+Z&ej@7DFpEd$OkPMHD@0^fP%bn=U|XyF-)Dtc_7I^!){75y_;KrdCx8|JSz zqF$M1CR&i%Dqi_PDUMY}v8=2DWVWXC-*)j$7s;VKjphN1%$i`5y=tO+Ar>gv*5Rxs z^Je~uSLso4aqns`8+3eybEOUto{7Y?PBRfn#| zR_)rN0*|Mo(+(S#$Ne}DmB}d|AiY75i)I>)jVLyDX#Ugbt2G1i&(l}RNH&~V>16eV zidTVfmg_pXY!$V9)l~Gq)>+V>p<~D2M(h@TkI|M%Sm}$4nz9z zsh`!kTn_ao3z6qc&L!VH430gI;T4QlpV@*f}r}}!-+}mBfFH|_Jbdy9Gk0N z<83DiBfm|eeJa0~HZJXf3VYO_h`ovjtIs<)Kmgg9rzqAtUc-3h*oDHDVK=g5sJ(-O zDhaqL<+#1rk`gh*9Motx+Ox6f#*1-tUaL5Ots_8_^}hk8_0P-k&ueqQ>3rz>w)aQ( z)MxAwCsw`lgzBTwAQ;H^1QnL)N;NE)b!xEwc;791L<=sKL}~ z!M7ysNGF#Ep@I1MIPr82pghxA`ky?51C(c+NkPw5| zO%Wjei;0hN@!DAN-Leb?-pW=B55+Na-oOg7!BwJ#`;8t5&#c9`h4G>20r~ePJ38y*7*Fm0((b5#-u12&#L5qnWGX!!n-dmz-_O z2<|0OvgvmV6$h<{{DWsSl6=s6;gs=2txLGd@_+4DW|pQ#4!6r8&+*R$UuH&cu<4YG zr0Nd3KclT}V0Rf?;C2|wwK)w%?;RpCYj0o=je83kChkd`b!v}f>#8T}YlPM?rto3t_p5mfPK&x0ZElgn^4+wJ6uHMB`#uyD z7BxI$)Fs^{CUB7Dvh8WB{aRsFe9@$%}`@r}&K z_>$SQ`L*x)q=uTdIHMk={vO7CJw3T3%eXp~gzEQAy=Sjbb@JUuTt^^x$ZM%l3A)o{ z&4(RG@O{JUK+18{k1`8UJ_Hu#DAyQ0b6^=^J<1UeL0H%0BSBuL@e zUBxp1;TaepJmW(kny@oyLS4ZzK7ZsqzbV+qu0A!NHyJNo^TdHd@r$TRl5D zDz)$(*>tXUp7B8UOJsV$5_iH)mS5=n!RsG9BhiCu6s&WJqO9I*402wm?niDt*w8$R zRFBWho0ec12Y)2rLkWb7JmC1+ffe&O#WmB^Cz->4SYog6rm3Qq6cr)H=exu zH@K#!$l=b=|+;ss0K8g(y5hC;XYhZ>lHhG?eQ9=)>@dGxzusGX_G}&_-)(<>bccVGYyd32 zd%KR*U-v57!p>08$9SY2Ni;k(bsepBbb*(&x$IQW;NPZ{wqmY$WYTxJUVy1G5^rB9 zEd9Oe-rTlmI>pj&4>@1Q&9RJDI3ZlLpK2Q39zKgQQr~MbVb|C;rC(0(wmGg_YAs%R zW-Rq-32^PJpBN#GFt-%ord{ZgsjlA80ID-xn?uv%O{Z=vV+g*8Qw#3$M#h*XV}4oR z^D0n|;J6eBn!U&IIFH8%Y*b&j#J>`=#C#f*P=JiiW3KA{uN+>_rw@Uzj@a$Xy&qoR z9Utf+>oeyOBV@*4wX4Kv=J#JbQtM(3=JGQ#>m?(13cYp*VVDuX4JJo+gTZT(-xbPn za*%GS@ua{#d07J;<`RVixUW~*NE|jWXHFqy3AMNbrFqkN4F+0xLgXIXFHNy8@M>{j z4^y-W>|v_%|Jz_{dA7Li@#Oyhdk&)+|NoW4kp3@+aqR`PJU5?z09FG(k9B->0|7OL z%hxMdwJJW}Kg2n2wL5%kO#X>8AojxhK`$lf6y&Xuf>>82f zI8`+A-cAl=wBOG)k`soGL@=2#VXrPaq8$5rdLBO*w*h@YCXgNagcDDIhE0pF znaj)L4&-+LpAUh9@I(TJ+A2~BPBPqpdKSO(QdS6-)1=$Z*mxIyOM-u5<6~BF{4y~$o}t8MH2(jsFCCiz*OwZ;b$_FZ&l1q*!*uJbNNh-C zrcGNWhL&Rs{&?I861MN_(!)>XePPVMh8*zy#`{v^!G!ZX=(~{n!F-9sFN6yh1C6x0 zmFTB?xRt0U&e%yQz=;8vj&Qr+eMPPj`jU*~$h5yZS|olB=p6Mi)<1=`A2dVG1eP&6 zc4Gr{23@w@ZK^Ny+klbK_nIvNsQHs&Z@Mi2x98_3fh*uS_xge-3-Ek*z7XN*xz$|` zTRzP`UYF71@gaaA^wkPV3>TVJ3XYvSL<^Dr6JvS$n{2!lHfP zYzWKZED<^;)=u7YTB1@41XiGM2g?FEJtZp_gi)@SS?dxj61eS5mlpeVIZ0k7#yc!0 z2gMIcmfiF@&y2kT4RCWxC{)1&1wO&wBn7*|lM!X?)6R8P+MwJwow-BO$$!5w=9-V* zkkYgF$)lwzVSr(6$JCRJRo4o0NkMiTg4alWrA&#B(!T-#eT0Q(Ujy6Y?klD5nf=FYwKO%z)m}Mf(ZPsqi4kynTIi$3zhRZPsD+E^;)N#FL2(h(C0r#LYxBwX^Dg;R$MR=V5mYbItW-2W zRk1lI#zgUJ`BR)Jj~@>hk27;zE$FrCP|3Hc(>CVg_1%?zcEztWJDhsevSrYL>P~z9 zV`muI!js6sX3m{PXNpODOy&Xggcm&Tsx#cw=JFBC_8OjDB7FG9sPd|~4rBxH2nseP z=~VFUjFWS5hve7PzBM{IBHE8)sydx)w{t8;bmWkkCaHA&Lt&9$=>1tK@|MXPDI0_R7c9e~d>^ z-{mnZC?#Mkxue$Z+%Hv}ah5_4!9hB2wzbv|@1c#wNm zR!7|5e2pMl&c`zZM5Z(p*F0HazhM zEi9wyZtxNwh>0edSk?gDp1 z!a~ZdVwis?kLY#JncRe;B$p=D5XO%U2VebJsm`+)1)O5S7)c0Y5-P9b_-~2{aw3UR zL%BbBA0-Sp#YFLMiU~{w^eN^=N}aaT(xH%xX!!tRUmsPdm$Qu`H1~u>7Q5}KT0-?B z?8A5Y6CO9qGJHQv3!S6cxk{D10SV{-rkFG$fKyDc*eUUv0)5P@WvJltCSu!)SK^%k zl-lwv!&1Z$BI%Cm32)HpQWvs;gq1m8Mr1htv(s1iSjQZN8dHwJNj{(%*zPy!h#u0@ z-T1}|g{f|hV-PVbpE}j+9lkj{iq%^7q5$m&9phaDut6HeIuX@UAo%4-~ho`L9-y{;+gzBolNWZ%$G6=g&mRh(ll zd0q-opW#))5pbLCH%_R?8N`#W?8k5!44pY4AeFd1Ea0}fR4;T1ePWPLWW)w;F{$%` zn-W@<#&U8l1dS0z{N#VqpXO5eCdiQiN=l`M`=j zIN8ot)VR>i@B{y0kEiLgZB8$2YFB`)mAqN{@7h@-{8AkMGJsRG`p*X3O93)%FsgAT-UuAZfWBg2r z#3ZG%wxnw1nA^8di)0AF#(o?B)0qTj%+PRocnKPZy6ZVQ1(Fyq+cGGAJxm0$aG^A7 zC3D@xd#nym#`68RHk2W?M%hZz{ZdSgl`+w`puaXLE-IMbG2X8v`=>;>z17Y!p?#^0 zpgjo6ZOb5NrAzzhg%z#7Hg2A!ak zx-b!%;1V7k%8Yuw-5F*?P}pahs)d~L803*KWfBBu&zM9|i&F;0Z!9!th+m(K>D+Q8 ziH9)OAw{XhKutgUzbcZ)AxUa}#`#}k#EO_3LCV;Qp5;?FZ>m^}NgNt!BZE&Ac!4Uyx#J-#2KRW|KPG!rJ{6;=W(?OEOt_5)WAz1C`eglUpMMf8lyD2ws zB9;FGXtZ}TM9d!BVOs-@Lj=r4f!q3v^uH=@}2t8CWzGP2xd#q70OBHV7~pJ<-PD;g_`o ztqp?-@;}mKWVX&uY|v!X-0F@^K3OoysAuu)DGST=V0Ip9Ug5IB>zpk#qVOBF z4>rcw=>3KTPbTluV9WT@rJ~hGIZB4KOlAVfQue;*BnYM7$@mAa@7meG*Z8Z*f$oEq z{{6jiuTd!HK4F=Vv58-x(LfeL1BuO^S%$Ka4E9TAz4To3{pr}2Kk;{CmL^(Bf^j{UE zO#W3iqZDy^GtK2EO%%aKTSEu~(}}GB(Fo-T0vLl&W_>-ox|M{vfXYg6GkryARzH8u zQ!!;p$qQN!=CRZ(lvw*uppj~U7b|Q?Nz?b0&b!oNfYJWYL{jYxs)|108vnJRu(YF&qY8pwo7t8P z1$Cn+cQLgO1Zb}Q0W^oPMFNQ~D(m+Xi`lr|IfhjVUqVM*%_L5Zibkr#$}mV&P1h{< z@?2%tEj`;Va&4ayeWZ1GSi^&1i6ZD97nWgB5)qEQ94*&CKIFhjyH@LGfg4JZ4ktas7v`EB_GNS@p{2EE_&%~pH zh>&97*_Mxh{>%_3r3cWTfoM6TTGY>~&DT?Vs+|=OMdC9kYxbJD8+{q0`pNADI}C2k z{C5Q_s`eX_6qNkSECK`=DAylODzC#@Cam3~ltIFtWy}^HSpy|MBay)feg?D|PY2v% z@RdalQ2qk4?|+bAZ3hUa`MRegZSCM1j}@b^7F zyuawwr)gT!%r=8lX^Atr;p6>IuLRE$ekEJoCXvjLv;@duu?Ta|Fk_|?`Qy?(I9)U2 z1p)Sx^%z5j!xNyl-ZMf!Z~e4fc?J9F`Z{7KOUY~E!S7xIg~NO|Xa>A2$BWf}Y|p8d zjU=eJV9MO>7A((lhIzSdW2iaL677P^DjI=Ui8hdx@jm>ubOTZ1o*E4z=fv(e!x7SB zQ{f|1FB=$W?W~(>bzrP1{x|^jTMv!IjNDbsK-2j4%?2q~IY9QY56hl=$7p}gbL;Ss zM?X`O%&}*do$6*B@kfGVe?r*@NJpdCd{#%}z7e)^&0CeN%Z{{4C1f{jEquMbAgsW# zwH9I8g3{HJVVBrop+sbfS?FI=vCe`a+m-n2r^cBr0e{tfD z&uwQ+m8kKlv6d0FS$`$34L8wv6@#-v{GI;2ng=WhH-S2v%tv5c&lO?MD&T&70JNTX z<|nsC9*u0txah2UX!M@4X-Wl##*MN4??k$pWhkB3a-pqpJkJ~v=3m%mFeKPvg$bBU zSxUOuQ=)+HBm{lw?jBtv>2H{{7R&1Wel1bx zQyz7vTXexYb3-jOm+m_-d{ln-(UV?~j-cVumVuHo2r(jD8jZQHhO+qP}n9ou#~b~<+I z9mnJyAJ zT+#B!b(_#`c~hbpP1J-=nrDLY)Q0H|E;@2FS26VOyaf~pY3%_6GYO(Ad%lLSbuF$g z+(#7A%g>6|Gd5VEP|D_EXf7~pFntG(^KHhU!88SgL-p5;66Uo%2E=M;6}3kjDI4Tm z?0lQxs7!aH)F&j!TS9dnXn|r!n4Sbx)J4b4eG7ZLyWeZFOeV=#gKwsvh4(k&Zz#>T z6h}sNIm!a~be$>!ewhs^|EfDuT05d`wBRi%m|^EJfG~XuV#?=Dr#oUqR)BsUF>+v6 zLS9C93e5LFix-1qIq2X)2CRWtVsG}5o|l3`hS#yv_&`?OlSq%%BnUmQ;K@q?iHQ?q zq&4^f0QM91%)>bG+%9xwO)o&nAAQy-kt(&nfEQmAemNHyW1||yaY-%nDs=Cy3atl0 zB*Jhvi3pd>VGIPgox{o>|NCoa`OdrzCgP4;iu?B*;|MC(9ftqQeC(-#N-+M5UUGd+-Ub#F}v@t8F0Gk|ZYxYd7ExLj{kuR*-IC%dw<| zSB3FYl0?;->1K=@*WB`+xgzesv#WO&d#~I_(rR{t68JC%{ya?ck~=60#?c6uLN7Jr zv1}I|g}HocckW+l>BM`2|5@u%Y-1`zo~k7U-j@qk#&FcJv=>ZZ$}#I8*y+1 zn)fGz=rqqbq~ULd;;!Ata{D35l0tC>W?q zb1=k}-LzL_d!U0+Vd}Zb4@#C1yCt3j*8IQuiP=jE24Xv>F$rf8WLPalL^je@IOGOCjFt1lT!OlY_tlc zCtMd~j4f;->YQC!Cl+St1zLc<@%1PPNpLSV7`u&NLn-a`PE67`CMyQ!xsK^FX=_K2 z%09lNNDBpx6Y_^Z6^H3C5`z^dnQ2KgQ?`=hCE0}TNV}B3*`+-XHs;s1tzH+dtqORogdn%ebdTc6B z&y9qBDcdx>ysNtIO5A2_s;YaSesG-?yo~y)$_{u{Y*dw8`0r_tOxZ*K{Sg@7eK{#Y zl=xGUumplRU#l3=G3=vI0JffY2-@>rmDOB%zVeI+M9HnETV^MH!N6;1BKFv}EI4fQ zc}gjk6+K&(%ZN&uTC!$DQGBn|DM<$U7!nlTD0T%f*8xSGa59O*sB0Br>R5w;O3upP zkTSxby-;qxnG$RXX zb`Zh=c@egKq85*raHw}OqL7=QuF_m{LYt)Lo|)4V4FM^F6cM~^C4Rpp3YON_F^7EE z2)MVll;7Alqalwrp1U>TkmBV}gZp0p*!59odD|&UejbQSdYO|kypX4=1!dBrATR~Q(^JL7XqJ{zVNFt$xCZc@as}{tKa?M{mjgV%^cQ!o zz4RDK6fP=2Rsq@KI5J zqo`o5GAGo9mX!|cI8_dX3jnS94@zh21NED3_Lj|xjpNM4T?=)WZHc(@kzylECXR<0 z98mnYX8AnO`c)*P-U_Eoo=<&((I)Y3fuN{KbW*miN|5Y$6hudp7fF5%s8)w zA*f}0B|Y}13Fl%PDWjGIyq%`S=R6Tgc>C?Z`LSYmL9Se5ekj`YsZBHo;PtaL0s< zgp4d0A#~9j_{rRs{20~xbU^0p)VU)y(O>ofJ+NwA8}pDpyM1r%!uJ_~G@qA_RqEfS zJt)`4ZJC<3IJ~jT@!0eL7{!%apa>y_nMXurRsVPS4iXic*MQH0Z^}IZ={G5Wb{{oX zY=}(^`FQK5km_da0V7BFUx*~+<3NGco|=xKuK@eqg>8b3AZYu?@{hpF#g%LQa3g{= zU`S9+m*E0V%8|{w6Cj-qg^rl+01gJ~DCvmb`_7aC_Z&uqv?Ld+k0t4jXiIizapj|w zD7|$i*ETi_&RTHq14{;&3WJ^E)j)1Wg=Vg{wTU<=UbybyiyBsi%DN~CP9sb}Zd8{= zj*>03SD_VtQ>3=OswSMc(xvYoG^T4H^-D6Rf*s$^F7zqn07(rzUVnaU!& zAWk$(*c-R7&NSK;$!Vq+BMy1F%|+qRJ^m@`^BgiaAZhA8kJzHCj+9Zx z>}M=cxP>cJi2H{y`WORlSH1wUxDEfeELDdA*SPxc$%*b>)P z@Cv#VV~6R9uofU#XwFjQh_-3X4r~v5u{hYMiP`7@@d_XYF7|)i;46s&dPbKfaCK?d zN|g6-;;dSfsZFXU{9@&}cK7cBw~LUt-HNm^2Plti7oa$z6_+4(GH&_Wz&Y9nd_%+8O-Xe!C&gIzB32w&GjP#h1yb|pqpW!$;^`rfr7G=fiRc<6PR*>S_7Zs$8Pif7&fTALwLE7VhRXqz9n6p*-qcuP7rb?Xv0i? zWZ(W1vy(L?k^Mr_d)Hp}D|MDEC;(3BCdZt-P|d6&0}ry$UpMs&YK2?6t%nKdraU0@ zWz1`?m8-WwW!CP$7#%lsSB;&kTe(43bDHjD%n*2_oBk1Hr?!4;{i@qJju5Tr(_c$$`v+zDf|C)m7czWsK zIG8&y0&l-_Jb0(7wbRvnF|e?EwyHa`7|qfxmF`NGUAv^4a?i{{k_RpLX~RK3L)hG* zI)v>jJ1GMtD+?0SS*P&=BoRf7+d#QCfiHUpcfQ<|h6bVSzaQX*142hnXveoMEX!!HSJQAi*C$_RTEo5=T!R@{SMfZVgir)Huz@= z_fkh(rOYNU`E7l_MMJG=N(3*HTCR`XPts-j%+4hX2eRtE$~xIbO^aSbZIDA+3Qy0j zsiIot`fDPnYf06-$jP`OShb*SsG-4|$ATix2cY1=GNg(1UF zpuMS)&XsLzznqN{MelMV9q>;bWWLExPIfTl7ZO6BO$mAr^lM0!UZY93C(mRQVaYSh z_?*rGk^vidM3)iEeeS|qz|<)FlI*f7>fYUe`=>wXtv+a6bkc;8pbC~9=#pbU<++FR z-|?$X{fPR(zj%txYN?^eJJTcVnGxs~->uNQUB=eg?)s-}HKGrapjD}Yh$z)R2L(4- zVxs}jn@bGY2!#5Cn^3EV;5b&q{1urZD*cBB{!t{gzz(D(|7Kqcr>% zyJPFL0q~U|i&TQEmfp%J>Xosjp)R@-Q>_@b+vwyuIF&muz#u^48kabMO6kWzv|a8z zO@cPosB1LtR(FGUb$YLbcdi(7wsWV)`KmeztN~fp_f86`n1=AxH{9DG(;N8pgx#q1 zostkovRQKkmWvP+D!w}MS#J}~+wa5V1LnCi-S78PC-5Y&kR8>Xdr-|RC8kdi6@Fq? z@@$Idbna*$u{J3WIbSR&K^gp|Z;&h>J|Puli5lPe{Q(4vq-`qkFFc^qwuHw<1s3#s zZFaevt?jwWnvQ9g3rurG_Hz|75+71MR*DZFM(Sa9u4<;cybYE0lVyFa;9v1y-H`yu zuBF;MXeZC_wVf!y*v-Svk8kpF?|k3?+#CJVdu(cWE_2HrB)C3$-!mi$ zD0qhAX*;B}Y#gNGJR*0J{oEI&p`B26-sQ^TJl3)v2{VxXNpc^SuR`wuX)uwBGPkOh zO$6vzBb7l_cxZ#W)Iazqf7O58s+#~$efGD#Sbgiu=kxim177Z$U-)O7cb8-Y*5JPn znFG^$FBk&L3X-;*fE_oG^Bt&T0u34#w5{RTu}1W!D5w(i=PU4DI!BQro&neu8CK-i z^2wRSVuE+oijKfId)afpRa~%az=yR;mj0|1@Pu-JB?#Vh0MPci zeqU^XY4sUlnzg^W>;|qJlQz>OgsX12bqTbZiT@NsYW11{!%Hk}*Y`F?)>UT%*|*)QdLPci#zKw={5ue~u3@Dx!`%7o^p#&m)7e^VG-cm%%EIQNtj|l6fN>(c z4hW)H3xU3KDu(k@?|~ETBjgh&tR5CHN&0Hy7Br88>I-XJB+6Z4o+&{C%F1Dul{7rJ ziU*GD{9WCIrS88uRyl6ivn{*iRF%6wN{9_sp~|*e;Ht5cR^Uf$E(6QO zv(xr}?4*o48?Yu$vwS7+LV!RKk8g~1n#;1#)_WIa?(56>p0 zRz@6_lKm11?DXZSsE5-Qw81z8&j{HpEfqIxGt5^oD6@<%Frl%P$JDFQjh9sTYuee7 zC(1g#+LC;le0-iP{(a!*%r-htHEO&;+?c|#lXkY4wjXw&xLN;IO+PmU339zH*j&y- zk6u+cUO9kY?gIdzs$Z<_XZUFzEH%SeYty?kF6i*3b%bK#stb#fSEeSjpvPoQ;q5ci zfNhejn}WXQB#%vT|GIHHlrxsxRbDUv@jS?T3AOp&eFvZix=Bw8Ay5bapX z%f%7x)MQ1U-MlwjD_^RmhKRD?z;74?pVX>ORM4c|THM^vY_0oR^|rtm>K8hP`?V)c zsToFd$;VLtRre^dYE4Mod9Kj{*JYcTdaLemSjcUn$diRU!VR(70S@!0sA8M)sV?ZW zd1~|VK^BmEf@^9Q@!X-o&rpvmQey$U+ErmwG#bSxio4v zb_IvI!h%N%cKvOJ%?|iJF2EBNcV#>d&-%*wPH!maN|p315seQZC{@U z@V@;YYRVBXWst`2x34}`=l%tXgHk{ddOB&fpzWNjIjJcLl1-%G15IKnK@f3e*6B4; zQDue8I!)XO7H08MR2B zZ9+UF)dcJ|uXU-?wzW-?T}LlaecR*JDbO+|15fR>$L_ha#KXTBUt0ilc-5*9d0eGz zU)KJ;s&o0*UA+#mVZ$yQ8S6?EYFBcFWKdxQ;9s zr3U(;l80OF4E=S@X%GWI+#RU7EuFW!=x6xpPJKOIW?h%NTW;Akz|e2cU%Q@f)9$YD z($G+b^J{9{+^w+iZ9(PP@U`u(ryi?hriuG56?OLMNUvepH;FEC)I8B~eFGu^XP7z1 zDyv(5+lz!4&x^MW37Z0iJ3KMsU*g=>OJeZ6M+(feB@>!@CWMDP)E=;%3^P8kCrdO& z*L*O#=KnxJhX&O7ualaY388Z{o!@EZikc-yrU9VGGMxi1XS?9nQk%%gt7Ew9D_wK@-ta<)Jq~)Kq%3WF-{b3rf2LwMvK=1H zR14ogTmDtYNYg_Sk9ls;P5gQnH%Y0!1so22sox?5Tei62PhA1#+NSKpHTE!6o!WPZ8@syDm>1_y*JEeXo4t8pJ-iF2aTrG zLCPoTkZ(SD^IZl4Dj6_RE^a*J&u*~F6|uSI$W+i-uEKPj4F5vHe>)?^l<(|m)R8k2 zUU0&s(sR&;E4S2?8yu1; zq0EE?py`OS1_|~d3YjEeQNLEgF{jq>e*{3hoBZE-y?g%duZ@lkz2FyI-bhrMA_c;)L6q%o0V+`to1PyIN{ZV@hT?uj!;7h#gHJVhxo-AhS#C*y6xt8Z{i?kHYptL31!ReY52!Vyk=r&C#3< zi3mcc$brjtkZ4Yt=xVU88w8fC5NN%&99G@+4br}=F}yu9K{@u{7-00LK_nXTsRb%X zm1qrU(r`nSpgQd%rV~fY@wbd*#PkaKj3vu^N8&S0e>=Ox8iElTbbJS~9MTGj5y&Uk zKja(eClZJ3+7Io20oD0{=!bo}3*(8m{*M;@c7P54f7@Pf7Y#JcM#QMp;-FZKfyIQx z^rJh$v@}O9<9J?s7U@D@&WwhC2dz2Jhnj=?*7w*h(75-o#K{^CxFD^GpgnXUETpJe zr%l7ZNh|ybHX_hs?rDy-l?cf=gkxG`qBM!vYz8b?$gh;AP2UqdMA-0v9}&0k*-hsz zLOoG``@vve-R$#jivj)o_ch8cf!#HvW`UP$My+^iX_V}`Vn4Vg<_iyq>$h}|2jiex zC*4Lh3Duy3ux!J~T;$B0mu6;iDT$DvgerxS1~a8j089&U9X2g1(@?0Dqo#+T(8Gus z$t}i|$OOzCF#?eb4U_=HbRgu|`_UXHy7K}JcPGlClP(P@atYeEqGW+xawY%!wE<;?p)YYb0vUapjNlKKS4hX8(e zJilVUnoO2@Qiv7G(Ve+Zj$G9eh&!(OJid+WqaWMd^K3uzx?sjU>YC(PXrRlL#n6<8 zGbzbiNKhvyNKTC4Ls%qFOvW{;%KbRVqoDOiC||g=3Yv%2H-s2WcmG=(z7+9qdiUY- z`SM-#L%kGX5h>My2>HNbp;6)b*mZgaf3{$x+kj>e=fxn zZ?xo(^S*gFF``6N z8%Kz+f>LyBJ^Q}=^9=Ot`6u2QFzkAkMKPyxdv*CyGQk2v!3_*|Fxz|!ujX%~q7s3c z-GZe8#`Wm1svx^4Rcw!2d;W!uO+&#yxssP>5IQSO91jj!H|n}-Xc^2c+sQHwMrWYk z7Yw|mWgmbg3i$C7F%qHph{u+s0F?{iNpTDdD=jwC9hI5FJTuCB4n%cp;I4Q4qK~)g z-FUAUb?+01JFA2v%C1$q&M3EhkXY+qgyeq)L|kNR1JWM+Pvb`@sp&Kp#q>I|{lZ3rbSnbel%1_+gq4x~4Cm)19&&K! z{|yFuPmCQFAAUahbY(q3!x{YPy;N!K+7Z*^&&-fmE@}frTH+(Uj&t- z!EiZEBdOK}7TO|90+{8I#0+pipmz9F7!H)pd|@a8KJ&m5X~5YF3wv(Ip&_=-#tDc^ z@Gjw}|Gz|y>_Qy3gf z_!XsK#OTZuCB_6OppQCJfi*~wlEdnumldQy@y;P2>h*UOgdRccUXD{9h-%HRRe(UYVPkQjd>O;J9%`gl+v3Oht)HvE@RRRKZGED-<=A47` zo*wI(<9;GO_}j~Rl3kJd?0rvOM*$^~KwVCgCzv$}8R({h$u%Sze>4PEJS$H4^jr+j zSW{)rcqWV%Fy|-OYJ!9jT;4nw5WIL3(y(I9j&f*8{4eL}uVKmE6xKz-k4=-xNRns@ zw`qeIY1hP56~|rOSo6lw`B@fOaWFH{GA-H~o;H6yfY)w{qBLhH5E7z>5MT0^a}Iv7 zB^yuK3O6p{df<19gUUh3%!>XYF|@o=TL{OF%~Y!4#@UFWq{T<@WEn8}@SJ?)iQwUz zMvnP$Auk@MZj`f5Q3P0d#}O#o+?RR&dGoUAfW~PA5%>KiMJ28m2jSckGIcHhOG$WF zxJgLA&-Z8{36=@1IFa(bqA4c<$Nc#qA|a$bR-|qTB4I`sR4|3G9*7PUHLa#++)>4n z{ybt2o3=v7*X$^E^kvVGrn`xJM;hM9{n(pF#{#TOGWsHAP?DRX=(Ymu+I!(9V(a%X zx9ZtMp9EKh+PL%}(gZSLvS6nStrqXkNux2{#8g-o414HA zCB0=ETXI}zaFSy8S>_V?Zf38HK7!n3oazdbR95g-OOVBEvYqzX>ZdgI{tB6%*BBPh z_FN=@T4pbtlsg;4T|sW88^Z&6Oy%A$Js5*Y8E4w&`J4F6DFZ^mn2H|Pb=I|yp5hJC zH$NzU#bRdUk23_HR_@$>R0e2l3T6Onz@uee$U}!Y8SHy^Xc-yzC@e|^B^fO=KLx)U zFMD`VM&&=JQJB_=?7r5`W;MfZ=K&<|rJ%S_IC<9+e6_w*ZtK1DJ5tliq zs#TZs)p7VR!A63wueUTHBSo9jY4gDcasOays==p>AhZ-ky;Gjoj{d63G{pV92go%f zEXNxvB#cAWU}v`oaAZj|MkK%%8)Ls)(}DO}RFs#8A=pDUdBV8)hXv8i>BcUTPs1|Z zdxs{<=km-cB-g%K%Ws|1M4I0@vZ50;CORpue>t;tmQ!zk{__App50jlIUZcVgx1UA z;O9)#lh^MTHN_xC^GS z@;fBhA!4U$&Xep`cs|q;N);dGjjl4e1ft>ZKrGNE6KBxzEe;J4j<^6Tc#Gb3aCAsg z`t2@mzSl{fQ1ldLmqpXa%Q=eZS~1h8u^YFkKKasiDh0ZtNpb{`CP|BxIO2)kE3Oly2k7z| zx9uVCHDR>B>5uBDI5U@=&73Y9HIyY|!R(hVaR<&dcB^tG$|#D4B4(KmCpp(3EPeiE zr83*Ww^*RBs;NiGTg$b_2|V)^2!bJ}7TTMH+ReW(5h0l=j*1~#JZ)jA9R#Lx#AS3 z=LvgCBZT@W?0{e?oC?h?C8&zV9eai3?G8^Gk);FQDDP7zLvvOW%U&-q4?UYYHb>$h zW}PuQ=5X;Ps50Ld^zj3@jQ_}Udn-1Ft#HT=*G$xcUr5FO#;a{TN@BcFnLtvCu`Ge) zXHd+Q-LA@H8OCv4Nod^8dB=@A=M0gyH5{t0?z4EY$dYe)F+CK)T0Q&p7DR}PUsl*J zw_U%7KG)&YXnfCBWhZBdEVIc5$C7a)c=PV?XAlZ2K~lQtTmfBSdWmpGNYI6xtB)D0 zNERVencG|#klxg(ekAs;Q9n!!;1G>lPSB{uA)t?;FJ~mzh}+U^NRksjGD1Qa8Vj^) zx8A#4JUQy!dtHYV5#iC7;39}qi|3Iq86G(xkR1i5#Te*7iDq2KLW45M!}bz0SU3<# zN_8?VS?07n4~93-hcaL5a4mgAC{i2&YlJc0k%<0{ zwA&!`ZnUyQBAbo`unSaEIH8z@@dmUL7zkY&R};xZJMPKSSD;A(O=V-_`Y1p`zm0pH zV@hR}8^|PwV$0_^BUhM~Me$hrjx5z=hW3h4>*0Q4#_dO8wX0*`3$kO!CMrumly^*v z7!2%8a?77M4y&J3K$VZdBDRyF2q&Famw>nwO6D%gl$(k>G@+>xtX(dolZ$A8J>M%A7^IS957(*w6Oict6Oiys$C70UBt&@K zqFK^%#<_Wqm2l!)E|f)>G1M#B#2p5<@p{F5q06ETJbsP02QG0RuqzC-8Qx8Y0P@7tWb8~0TJ3`GdJ4cC!B_q)~JY8jRSIT^sN@^0|*IKJ=;T{$uqx8Ir z`<=AiuM^0Q#tgb62(HxAO_JbKF|o`Z9K;Fp`MXb4$x&lj7xWVMsP%-DX|-b_cBZoh zO5m~mj2qixf*S#HI|Evb^ia|QzJE;^6y31dc?p=R*-&BsrIfY`2%%xZq?uGd5A zg>UTMbirx;l*7XReN1g#JS|b*`jp40??wLOMvjWMc+|a0rp+Fyj+;^_ZP>J9v9!~* zQE;27OVn!+^qiOLa0w?|MUFUGF@HV06){k-$L#7I_$}_--wSAC-+hKs;oTsjDA==s zGB4m3gi|&$nIC+?E#C!hSmK z+%Mv(`yEd={jBYVZn%Vgp+KHrM=2GXpS_f5Yn+yA0iWH-3T(17<=K!&mmkyHOaQmy zUd>vbZm8xst|=hic-25k8F5FUo5AJxG4AR#T!56Fcf%Wa>)<=by$!2R^nb?oIX!s1 znZN=)@cR@v>M#)U`TT2VzZjLAIY!m*1tbh-1OF|deqa|D`_()LC2~8ed%HPAr!U+2 zm0okYxw5$J@lvvrSKn;ZdTZBg)*9EP4u2c`;~tGpt{^!#w>pR0)$1`4%EiXGvEyn$ zNA6jjB-h5-bB9~&Q46jCozb5cn-F}S99MQTrDr`jSc{G1Z-9&=d+58>UahJi)9efa z3c^*iK_PeRBx!R=5WVLk7t&ptdr4Y7$=AukPpQUt#QZ*P9=YlFT$3AfFWzj{o!36M zmG%2GxGL_YLnPc0&YauQir1n@4O3T*BbyHiBL^IWOwCR=#S>1zlav-u6aN zT#^Adci+PG12`ZzpOs3q?V9KH3X=z3&;4P{QBVbH(_Me&S?~6|G)UT#n~E$+|cS4-q07~>R|RPQR08vD)TEJZU=6RtrLjsy#|*^U7^^O-!hfppNbHt z9;0e~p21)9YuP+ch}_9>kg~}8(RwCFB6|EYgo$Fp`&V{&j~+C-yM&TMpeQw>__2Ob0O_}1pV*gS#THrmM(VO*pP!{i7X-NI_R|i|#9KoDeRfTz(F;;#d_UB5{zXrEv z3hr3cDY3-nqD!A^w_$WeiI1C$uZEqA&&v)UP1TLS4_#*EAkqCUf`=zj9LKpE|0DKVf`Hx|=EqB-`W_mYx|B)@_(j7W zhz({V8s(W3JN_r+e)@b2d1G(OyXO>{3E5+8ThSf7_->0vH^%{?kml^(bPe}1A|p!> z@xOT3s&cKU0A7&XgQpVDGNJ+Oq7xauKnZ!Yew8@aPd}{LSV9-E+PpT=9xGaQnpEAn z;ZA}Je)#;H8ct+z9UHym^?s-O5MeGtu7mY?Sx}4I$xlmj{Lt-w&x|v0^%#H+2RZY8 zEC1Yg?BDy-7D)tH@YKgWEht@ zGsQ*J>x53zdebgd!yuGKE;H`5 znV}=^vaam0?Axr-(jq1s%9Jk$wX>#^HgNCOrNv*<%5v)3YuA8+dUPljP5Yzupt*~t zvli<`+G01v`p1ni?H6{2zJb=-;5=;fp>YrVG;WO(D9&EsA7dV7A{2`@gp-sE;rfvzI*eKxp! zJ-P;bZiVa1>)Eco->zRvU8|B0wJASw{ru#O`|aiRZ0UhFeL`?73g|)4ZPt|I?a2x$ zIVp$zfq%edp4oN-`&vX1TKlX+_@!~3(VBfx0_{htk#Lb-ebS|bxEMH>dfgqnO{dc( zN-&$u(-FYN4q>00HHwR+LKm1qk+(yzG!iRlGmZ+8g{IGY+01kSn+Vb!-@W(7JWZHs z_0{d%rtzTZFx$l+dj=5*^5u>dL9wM6*L}oy|C@L|q<^OagTb^at>aBN-93xj`7rtrcINqG( zOQYKp&~*YoSE8Zc9W(QvkM!4o!Q$XN&crJNLt?l7SP=O-bp$o%im1AFRh!rl-(L@i&g+UxG6_Wr!zeICyS)=4Ifo0~w+`C$8^ z%!CO**{`+~6X?iElVD0V8H_l|1jIwO=D$I2UBfJ8oW3Y-bn032G2+r6o|MyuXfd@mu={)YY%C#hN+eCz8qK)!Ou7y2SE>dIH^m zws|hYy5A(5zql&W+4bY>HwQheU6D^Ats8g08xE^mGxm<6$#&V|7K zfrO&6;}^MQ1;JHbKAk+Fd|-6p0B`>0=&rPb&Yrwn0oO`4u|?SVJD_VcXM%GH*3?M7 zDWE4EWkEAJ^L~ZIxQW$kxTbqQHd@25*~*30!ree&rEoJ{$FY5_(xRJQkMK`QZ$yv% z7Ym&BrC0~usa`Xc9YO~ibk|h&kb?f$XFIS^i>7iZfY8;^PSl>*zdUynQo$bd}{M4U1 zf0|u8i0wBWioPAG{Ws~x?oe&rC<2iDZ}jIVgsm2;BM#vG-l7S(ow#BDzV-9d4+SjU_|wMs zbrWy6?^m{mnI)6{M%C+W@8Y1Z!D!pJm%ha5?k_`VTmr2v z{6`}pg|yDvUz=G2_5QTN$T9zGg;i@G+8euywba~cIselAv60yS*GB3rue;Ts|Fi?W z4RRg)v5_o(iDhGI)hTyVyFdT5!t`&9PLDQgJl8AX{BdUWv%UC;-01joGo znC3^xO?KY3q*wdcj{_(-i~F>Opz00*Ve;8`_#eSvz4o6!2A&-zKv7OJEMv!7@CV!sTtw;DrG1QjEgwXxzhQ@CU ztoF-_NLry1wcF3{XUzT!GJnGCk{W_8DF^S@mV4dA_;7dqvf{sX2ne8COaia!@J2$v z^P=6y`L^d+x4-xI9@?@0pHi4|`f>}2Hs#c~+T@#ODyn~~=RA3zvHHXD)8CJ+EfXLf zBgB$1NCoj`HGNQhfkkDXbNa*2ch|f*arC~WNR%e)6_)>QAY2pWq)miW*o0cm*Cyqs zDY)y9_TMA|76YnRMf8f~{JF%n*ZNhWju5$zfIl*zWHxHvYXn^n;#0LyT(q zfSs?uhdb|Y_&+}Kmm6Sjs(y{?=Gqgoi>#t=ui}r@_xqiRu)eONUZrQ@frWrg_76g4q*l&lfEM5~Q5Ri8}rkX5OW!zVniU1X=ypdNcU z8`1`=r5_sU>zrnsbl}o@a?ReY75;x{q*lY=4KWtx1p?+Yv>zHtAweb;EIs*uX(V%6 zo{*z}o?E0#uJXV|XMXk>P+c&WZbN>;aB>plkg_1_-f~(wJC+1HBcuru%VXkU5+9T__zt?!T6k^o=51QzV_}#PkII zhv*X;=6xtYkrzojPQ^0?RMjT;qcXH0XunQXDU&iR=Ls3iqeGYGM<;R{nWE-o_9C~< zn6-v!j2m}NwyOUI)ZQo{VTjGaH>89xs+w+&YH~l1ZMGhEIe)3+iHN$x zl+C)>NaC@I*b~LqZgSn`2A4_wNss=Ps_M9pwk`6AoPY4*y|1o~nA82#ha1}I^?&MF zO2R1h$u+56<39RgjE}^y9B+i7ZxC2vo*L#s`HC<{o|0Zz`_}u)84LUtF955zMavqE zF)cB0_8TzTCPE=a_zgKGUU-TNB~*%~T6&D&`6vM`xl&ch>jfq%0eqbt6q%`LPYUl9 zpTR)x6s{8VUnLCR86&mB+DT2ugHap@O+c0uTre2}7@AL;swywgOW{UAVJ%lOku_1M zM8M?;ygMPsu!{kHs>sN^@Z}cp?BA0=UMz0!F(DR@=gFT~ob|85YEJ6g!?5co66P^Z zS4p4(g-6=Jd{;M)p@h`h$A?RuY+@orPr`MKOAReF2_=WrjbEE(f5uvvmjGT5_l(g) z6{E1zD&49Aw?F;_F@SA3l!->+s%~x)t=k}Dq=LXut|_|YB3P8NAj1S__A}zeCnZh0 zlgq@|*Pon4!*MK*f%ThZl9(Ramchl*Qxs(uxhbJ4h@yK8R41D!!xBoe?Wk!-`;)mD zXU-iV0#GskXm znv%&aZ7ic!)@$1|ag5yAgLiCb9LDJ--f80?r=DFE>>RX~7)wML9ThWs>?RuX0&)_~ zV@5`Y2FppNzCR$&J*6a8-$jQ$imw1bKKTmiDBvON{wN*Ye zK0|YGZiE{jt@avZUQIUzjyD(IR#rInB$4SVv5ZoafoWzkt0ujRr%GRyFT~GE@N3G6 z3AKE@o+Fxbl&Z}*!>4%uph&KF8=zjIvpxWs!uQ%Lt`?M)xs@c_unRl=TIMrxJU>sh zTLHZDcxAquxk?huuT!2Xf(8kp?Q&0)*+adZ_6BJ`P0BhBpAJ2}k-DyXn9ydM$gG}> znkylM5)ZU&PimxwU#iWOMS zq5wu%k^4!6{jR>XlB*lvD4&Bdr`_LX)>bZAc0v0?E3L^_#Hj=8V`Qq(-Ix^?=+e@E_#b`fWkCU5`GT*TNO)vMQ8mydH}-CcvP50Inj5KwRap{p@wd+!`n!s zGmA5mMmIl)fT}FGBA^|uvv*9dpPXoziF7YGLLe({K0!|CFLN}sT+Is9JVdc~U!zys zIkiq<`JDg$yugfAu{5ES z6-Q#Q95-%R=2DdA*jySKc(W`sT%5FQVv@@*muLf*j8`%nk^a+RhYZna#h}-^EOr&6 zSskM0O_)zwxZ-4Z}gTnIH`#rp*&N~oxj?iIKHAA5Hd99i2n=6OUT z8Zk37Gc%2tnVFfHnVEUS%uFL@W@ct-B#ofc@BcftlXBTk%C5x8chbkIYVGdb`?=P- zJ3g^p4M8}+q0&^yRfEG8)RpFpy0_E?3hoeMIVYFy{JOi>!yI&QVtua@i zcQPBe{f@&a@>73;0xs4oNDAXQv@p3KMZrCBtF^tMwgE*wZDJd0$GMe|8;<_Uv`R={ z=T<5?xnsV%&ii8}{h`L5RbBa{0Jstf`Ja_Y=rP{N53n>Zy5Y9Q!6-$a8FKRanZT7u z(;6VJ z`bee!rxqcbb@CZA`G|jR|_G|<+rqiidyV`!J4UJa&ojMIKIrHNs}Lb z?_Qs%&>&u=KXc{!q!p?Fo)I@8(mW-8l|~~_NZcGd=Au?2v`H|Q`TZ-(#;#h z%}Mm`Vw=qQ67@%>f7^*fCJH~rLhbg`$nBqNq=>Ax;b|k+&AoSyu?J}@|I-Lt&QErP=mc|Wa#-yD=F^Gsa<`O(%Qg$k zom%iPI$uD=Q+4^Kx|*xyt2eDx@)=&u*=l%eQMQ-9q6$S%>3j3zejd0K333;~s$k28 z4ypg9#D?Nnvo)cWgtP-L)wy^SF3pK1Qv(>PCmtQ3vWHi}eguoY_vpyp>n70^-WGSB zI7w2B^&p=J3aWu9ngn0}O+%p8GWhU-9Ygg<8ZD+TNU<4UV;ZbG=V?_mB=*zq1j6n+ zS-gu-tFpV7r1H_{zM7GY@@+XaOh@hpei{NqM?Za{4Kv6N=tLB47E23mP&eny`tJ;s z6V~*X99*<+@P@A3EG4`XXdal-V>ui=lcgi$G^|PW)X^1r@$7$#gkdHJ1_C2trVg6_ zM8aqcpiEY&fRV7OQeY&k%^mgQwWk3X3Ck(~4Z&cd5pI#>y`AqOB^VG&`;l#8ooLsJ zwMsx{?dm#%u09422>EaTzmdia^3D9^C>o8amLN_>GgO@5u|s9Na=6Uxyc;-Mb!~%_ z4*DM=VQvCkK8mlvNEqOc_Ax*|c3p@!+yU@KUl<+XU2Tt^pl9T7`(QA!Mwx z(*ltDjJO^RuI*Tb|5U;bv_eFttNYkTgg`6pAZ9xTKpL>?5ct5?_=@#9#@3;3iBvqi zFwobw9p)qxId}mx%if>=(TKh2hY$AHzYzeF+AMlT?L!X39J7IuA7HRsyYPB_Fy!4C zi%QH|Nfn83c}!=?|&f({BZYwHFQT5Ly@BLPbxOz9-ja&9+`=JN-!drKo8+p?nHIh~_v6B_8V;-+!^h;J<7D|aev=AkA*h$o< zFSDiVzGk({|9tS$v!{+U(4$9tjPTpb=L8YsQO^W1;{lEyGt+|?X~H92CG8+|Fhv)C z3eTC2U8aXR!Y5q$%M`H=aD!wo%(9G z`^;Dt;#>dVhO33R2nMd0?ZVq<7b=E~qKTO5Tv`be&b94LRfeVK%1xw}w`l?G)A3%5 z&8-bvH7dhtXj@N^!G`bLU6IL6QBKsw9-AyR5nN6UV3S-JSMNBdqhLITto;EgRj1R_32Z z7@`-S__u!=VOe>#mrh-@x(C$Z+L%|9JliWMN4|e}MI<}_)Ez!nvfaaNraAc?7d6{g zr>!hPgZl&TJ;0tkScz0iahZ|DZl3W<%zy_&Rben{40M*>#n@K+uLZzG5kQJW=2xc1 z2L}!2z2U)SW-?>!!YoTpr|(T_pjK5`WUyUMdx;$rfFYA&1nK^1N}K`nCMA`mY6-Kf zbcah7SKhYrV)!4pkw&PiEQp^AtpLYPPim87|X|3t!K2b;nQT}ASj?uBnEpYtkqN>_P;78|lv;mDGkUbekPF8KQC zjkG}wIrGP@5<{@Xt1ryyZIRz)mE;>u&O-OHO3BsGe+Y{MWPk?+O3*7Z7;GLHU`TXhWEl(O5b&@1KQm#xUQlL?Y{{iot z1pGYoH5?^nTf%sj`)yr!GYpZ`7ye#$;-$b)6**ZQY&^e?;|oJ6`nakqvQ&+S*Zpfc zmRU@}ew+^K>5C&C(;b(ObfsXx%e6En=D!SSel!2yf;8t)@aj3f2VP5EcH-oko+=JR zv|lY6YdckyC|q_#99C^4{0vt?RIo179WuxyWIxhUXO6$AgODjsRB7ZxOZLFJK_hiT zbOp@Bd*XH<4oG|YQ|8tiEsdx`>#bM*CrdLtgK_62l8JuAi>*ILn*@$V(v=AlGnhl6 zJ0nNO&_jw+Lb{OvJ7mDPq=XXME7Q*+wGsomTZDEyvt=b`vSKdBvoDz_kX=-oZaDjp z7CsQbmGnYTYLY}cCJAkfHkedKb2R8uIIf#0V{SU?;Do-iw^D(qehQWqKi2;!=%-E6 zdYK|c+1s|@Q+Q<58d#~7~tF3w>bs)`+9$;{Wa?S_Uw{}gK*(O z31Z+0(Gy3k@GL5l%P#$xN$>cWd-l$8A$VHo4*27BR40l4C@mz%#8#6d14t~($A&8= zU1yD}SwOuid_V@JKs{1)<6SqIQu3?7?}GnD9VP=Sg5m6WnkP4SM^loaM#6Z`xIQtx6Io*LV! zzWs~9sjRsuEc50dkV$*>LYb2cSLV88o~22)%O+{k(XWe4!pLzE8@1QJf&}80pArZ);UyJY}#v8ZH+VHH#(Sd zR&;rOLBBOnjf^*`S2`uyly~>nvnKJtynU>*)w!MAI$XG0t%KqLQD6}JEZSIJkjihr6asGSci#vL0B^uT?P`#vr z27fnQz540P!bXX(0iAxs*e0y`c){{w~lhfqj$mGB&$4-6vnIH`&qzB*wC?fKUay+^l>2g z^AXw>(WC|G7DwUmc(7Cf#OkqUr)pPV4WcHu`0g% z2O9IVTvBH&w^gK#1kRrlV&~AoeBbpl|9AI7VU9h?KCtlc7+X|4rvFookhFIj-tDy6 z+Jkzg-MSpooo@w*pC0`H|ANZ zOFJKKC%bv~&O|Nuc1>q3b6@DObtAA}?qk!ai%dOKkUJ;&#)bWv)#364SPDD#8Ltt#l<8aPL9v!qzaB9Y>C-t{)k$-cYq9=u z+Rl_&6HV$#h5zZ?+71^I8)7xII-}^mTybO12Ul4vG7F$jE6DzEkAETXf@24!!T>5Q zPn*?G>XeP%w(t6b1Y+O`ZkDD7XIZbeJ!Rgq@4H_P!24+6=gfc~YiA@{YqCRVt9t4m zeWY%XEEF8+p33%W_dJvP2u=vI9xfV78ENhZsDb{|SXIA}fh4bwPF=2hETpv1$Fe7Jt)EI0w5H!BvQhR%vRAmLpCZx7RcS(Fqu8Hc|6C(d`Up+nky? zCqFb+CCEOD^+5E@wGuB%bopf^bxV-`jTs(OSfTc zRf&(Ai;srAi_g;*KV{{W;0LuvAud`~O6=Gm%G(p6Hm{ADkVQ}5v1=I^Hg;9jh|Z%U z3AspOr<%#{vT6Rl1&sXRH$P(@B)+{EtfaI=e@%fgRM+^KqIA^rhnu#lP;i}GS5$`pmb-PI1 zcv``X|FtNXlh@lQSr`hh9+`>Z_*qKsdBlQ}}G%5>*{jl#qpK;GOn{x>KLjl9jE&!`Mb8=g>~!hSR5 z=l@6vQ}9`HI*3#MRSBay zKf7!q;#zsG)^)0iKX68R;Z|;o9?na%KDsarTW-B`ClmGDGP%|bt_zO^|2>#I@8)M| zxMkqcrxrfyyq>45`*FBX4k~J*NV}u_!wHt3o%~8SdP<{yL%xaX|R`7Fa%Ya`Zz^x1vEa@pcWFOTp0<@kOy@XaNcM6arWV|WZ!TN|J_TUZ{p&j70H7sT=iZC1zjfRKaj)yq?N+d#-@o4i zR@wCjU8gArATyJ!Pk@8dCD(~yf52mz^C!ioaVPviZVRTSau4{L?($D83|-_SKZK)~ z4A7=1*6;eUyI3~;*Iz!*?+|cv`l9+OaQ(IVaMs0>vj#IR#&)pv!uFs+qbK)#cHuG=n1>le)Uf0@<1ySE zJk$Go>3UN4Dn}=q4S^?hzpgm5mjOoaA)<`r;RaDRlu>=I0pDT0tSH&Zj`#YQdN=#g zKkB0#L7ge+^0jY&(d=XN>&xLT7Vz)2unWtx4MV~jR@dCCf^BcZA=r_D!M5FBEZl`h z-;c~>xknfT-O@~Abfo-kuH_!i7yo|ckjCTrobA40YD^^7q=1sr`4HLv37D zMQ1ItA6lU%o(iP^?49qEHZ&|GsALxWOpI zS19p1Upp@yk7A4v=O=R>wK?xD3rGBIV`a^Ncew7ZuvMz-E)#nv(|F9kPW=Wz%Lwl?XqoALL4_Wr%y_#j9S*J>18c1I{8)+>h483e#VzSq3&0_D*E z+f5A2f78U2{kw_b`8N}D23Q>UpNwMm{}YP&`yWwE_~16h?hNpv_BzS{uAQiF#H(eb zMjKI05(f6d#7|u&M9(TFdO)MEpoz_BWg~fky)enfb9gEZsmY@tU@z=3$E;zM1S{J} za=YfAUKpltG2qfI!nMTnyS`My&zS;70@$9rxAXADB#f!cL=nnt;V&l-se@mLdYrNo z{2v$tFTd{vqWC}51u|sc9|Zp99}r(|#*MHhp(5Zm_DB@APxVObXXJUu|8}AWoQN?$ z;{*HM5?{j6A?iW6eEvk;j0~^4nkrs~$SfP9ClG)x484em(Q3dl!prwQxLLq>bnt1N ze#ql1xp6f=pzD2bv@7;eU~T*Te~O7|{Pq8H6O+)4@h=kdzb=VMA^#VN`4@@#Ka!Y+ z|5Zth^#4am4CTK_%)dy?zevo#NX)-T%>Muqvk$n<@E|(PoIzR;0Kd(ZMHZ;s?R>r6 z_hb1wy}wzszu$d%2v#AX-prMNqMI-gCLu(FzAcj+Vjd7bA}pJaf#}*Vs8V#J7eAuD z+j6n3mmaIomM)}TP+VnOJ;&xTKIiSP(U`?)yqxFd0N{qn4+dgr_oX9o2?97tC6kP} zDla$W{+Q_qcOn<{ejEg^zwdBq>pT#DFMIMb4`hkMKz#I}zZJ?|peMAEOLh@0g`SSG z%DDsZry?Aclix;qnr8!o%~%raH6^S+C)nvZ86c*OB-e)hXFDjd5R9_{e+jC8PB7oA zoHUW0ZY^3Kb?`F|5~L^dTr`a7eX=uL3wf2Fo=T|Q29EV90Z0qh9+&-$ zzkVq@aUK4K!Hy)%=SRx*zdYdCPexqN0drn0Kk9PfCdTl%L`+h*telJ?klnd^Xg2_D zsu~_oi3Hji{plImEU6v9$@7%z`QW>t8+yU#801E^VYnQ*YE~;gP{WUuS9@S5YZ4QN zQk6#@*BFy+1(ls=h@a-^I;~m~h|-nJQ*mLzcL(W`Ivp@H%&IG`0oTprccx7q3*S1) zT&Drej<8s2L^1{@OJk*@{1o=PLPMj(jw7?NB|N#h7I^~bH)|VYy(~e|LKm>qxxgRO zM~%cGH}D3(X#?dg#Vr&S5)=i_RT}>W&nm&AnOp)#?IWBL8?W(xU+!XR;(hdy#9)2D z0T<{eqLcEHZan&91zb5>wZFj8br*)QdeK{T=#`%9otTgp+@2G3SLaY`VmKyRCQm^b zP1@fnVlFw|5qj-u0X*QR>cU+7+0m8QH=|LSy2UeLM13w*T)0r5PD>r@x0vI&Cldog zaGrc9NqU}OFbv_Lzwu4%4t~iF^sutp{~Nt-A;kda+u{HbxS;Wj__yU1l(T0+yphJ} zUwc?*p>*@bplYc0(n*WI5b@Qdmc%-|ju;>*Bq060o{d6AoViXQ8G*K4V&Bqw^oqw_RmP@@MyE$V;cG6@d%I?IMw&5Y(4=hWM>`bL7!JQM|~#L(-5e z23eJ+Xn?VxrQ6_fCk3L|`PFQ5u>-M>Kv?+It8I_8?l4jo;{og$?}R-a zGsvD!c0h2`STChuOC%ohbFBO%hKSDk2T{80w)@7l4m?KI;FpqX$*fOLOfuJ~zv(Yz z%oa69Pm%f`xiO}rRz+FM^-QX$HPPG&bVwH3(H`VQA?m`e_|xsA%GiggjD~95VO<6) zjB^8~NGMr$7^>gDO4GP2I73+z-CHlV1Db*r9aR!mCDX=qF)uRuGB%+w*>Fyqs0Xpq zpI>d4{j*FOkzxUQhJ*{al*KFkx0HtMP95P|9(!-bNY$cQ21sVI=5Alzga+{{qzN=@;_T^54MH0JD*vyN zRl%bu(iNTS`GB2X*45%DH=s6YRkKlf4^QY-1zt&z1N2w~3?q<~dBs&kyEU{?c^T}m zDB_qb+WHmUEd~iIqj1@*4o>zu!ZjR;uTtv|oH(8B%T4(x4Mkani27}m=I}Zt1@X^S zN-hHT$8*|>uKgS37HAf?@hEH$#s%@Y658xE7D#TKXAcCz37ls%K?fECMZqox3nGgu zIJ=X-YAP0rGu}Zm_i6wK-5&gK|-U8(b9)LHKHsz^S%9=aoGP zS5^a%&N5gv+Mm?%p&R$*pqT|3_rm&t?5vK~oR~kpT(e`=X#=Yjsi#9vI2AfVfRj`v zk#qcWdv1k4Q)dL>E%so3 zBORJB8y0vE^hDkPXam)-Qru7*W&ENX`OaW^!Qx|5%$3zBW7l~zXQWW8DNxW;>pcOx z>PIgyg(F??#%tx+d_&mCb*uIKd`2Q@HLIG_Akt23fAQx#B~Xr+CsGjI809z<20@7Z z+@MZmm59<+WC>K(++=XLWnvFQjVras`<^4dJBYd{k^4r4n1rcT9dcWXP6U}5{e~&5 zh(Wf;i@juux9z6hf!J4Khm~-oH+(VE^^)njQN0>x6h@Y3Z_cXf+h-b)jdtMT_?31% zc_)0CNrQ*w3RAbv^swGS3&v<81HT+~Ys~QcNq+xbjciY$c_j*_6W$wnOa=Nb-S; zN6*$Vk-dg!_bBT&g7+j(geS|svmvi=1b#?+_)_5sqyD&>mKudw)Q|+BCS4j%UniO= zs#M5|Y0%hAa7lm@b;oHG5lf!h(`(G!O<6RN_!Cu6^wYI`cMrx`bLj8cg-+v|CHfq1 zqPgvhWM4R9ep4tv-M}zofBt{6KmNQ95#|GPfZc<^52Xe9l{Us;ql#{XI~IYTp^@8+USoVCGLs-f%aPg1iT% zxU8p~g@8ODt5CkHrIlaPTSSAN>iF_=PFlT8GC z$v9FSxsH;kVDmubVcbN_bAlu>r3~Boo@L`&k@sv>aXr)s`kBHi2hgez zr`sie1d9T80oMSnULq1C`9;z4W|N1T{+N?AF%@G=w*jnC(M^Y3r2FG8_tCBs_!vf#(?$}B6$`?e*TSj;f(&I;tU*os*r4}G3$WH-I1 zW1bUsbT2r_f@jHih*&hG>1W=DIe&2!N(QOrcoYiwROY~JK_b6m1$E9ZA-9TbXefn9 zxOU=r0Y6qxdQpl4@@EhodTEFhBl(Az%^~fJMusz;_9a0}yM)qWbi{`M$uOaF78N7D zK`WP)uL9NaQAJpv{RiX&pKT9ReW;cMRU~UzS zAd_(@LT3=NN^54^xeMV4$GiXo%k%KQ$9qVFk(ihUE`fvgoWR|m>wD7JI#F5YkUrfi z#w#Rsy1DD$GvphoP||WQ)H*EX&ZH~ru)S!+^!gH3Dhq3s(pT}Qs-6Nn_G|Q3Rp}tF zN3trr0_sE*UEzLms3~tll;>(y>c!)#6&*bu5m0IvmJeG}X&#&hks^BDEa%6?EMJ9) zycO#2w@CC8jDKijgnztRn?;O8{NSszwEGMiSz)lW`fLnaVK6dtjjiS@k}2&1vt?lu zU+fWTi)=AVaTV~O6}Y>g|F%M||I@F668Z$!ZtWhIui$g&GST>&Jcz3S`0J$S7i>&j z5{}{fg)KQ&YNdg?{Tc)fFI!BjqnTD&T{hZQ#0c_`@Br9=qFF_j-abLdvs(7+*UZC_X)49t9E-#=EH6)My!dpa3w`qzL2PJRe9mwfdTz9@&fS>il^%v3$B515xGGE{-LV{aU@5ecV2w*eWIi2 zC8Pp#2>Z}XVr2R1Vo9>{vK$f$D3W}B_laU?0Y$yPdtFlg@Tg6N`jVYfh75fDNGf^< z)z51h9?JkP0Cz5iX6hIcKcckm5p8CFAMRDeg+6KopKDqrfrn6oF;0eTyAe%()bQYn zN4WS>?} zfVk4bUUY695oR5Dj_ZE5CwNf6s9IYMCi4wP~3TJEBuL?oEZOfrx{o=#O zu*49+dyLP!(8p4bNym`FJbKH9_ig7{HWea>Xy$ZoIpFK=63nl3Qx*){w|nLEIgkXO zBhF%aBu5e9IJ821L-b1z^0bn@n5qeBT})D3z7$c=X9sr%L|@%ue&;kr znl&oKa-=W@TTwGNXvI(D&{3pc;?}&l2aP&s-yr7FURYYhmVz#xi8}dZ=|s+zuC$bb zRp(%T+!KAX!Gxo({9(U0B>cxXjIT*>B8eBw{swx2+!uqV&T-CI1<7rM;3xlYa0RGU zMc_>us_|x1!p{rr(wx|z0%jP#r=|%H=NEwrQe8U=QYT3)P`c4&+Ur9XO@ zWsK>fotV|EkaOKD3|;IbxOK57v(SY93$z>mj@qm>YwZIHPkQS!L`9O~5hs!0Isfmp z?3(m?_*NhYKlFsy4C<$dYQk-D&8Zf%f9?EzD<^p71b;`8_7HCxpq={|_~4=r-n`iZ z-rD)OEJfn_dZzQ;BZ-A0vRIi5b{Km3A(1*g@Pg3EHWz9T&A3m zU~Ym_GVCY}lK&l`SPH(xeubF}h8&a|h98*n)SU_{jt}AqyXxMBno6S8%SaR zG=zPhLzHArRHV7pE!mB=wau)YotTnlzgxuW;aTn^Fb%y~~2py`=k5f`r`M6`SIsbjWFFQ3xrbA0!;&?vNqP}`$*R2Nv2CP`u*8sigW$ae+F%@J1F{3y?!@eCsta2P80hcMLPC@#6gth0(=GmxsH zF?;ORNvLreY1CS%`i+H1%cnJ2HN@0+2-WiL2zoDI$gdD`YL zSBb0R(rsxvk_R@kA`C6HZiYXB2~$RMigV4UXg0hS9G}Evxst!HC8%AwXf@Gh#SvPUOOAgSZfEbX8Hj3n=0pK;O33t?6RY1SmF@vQ!63<8o%B z(3YsDP&*z+;WtCpEXwPkZCE6cqm)`9J+>{fiyGt<--2r%V(YB@qVKzK>`3qVmd!a|oKzmXk0_>7ig7IK!FH2+SLYsmU$#+9PM zj5U**40p29L>1%%(Do|u{@5l2#E)+-_&@&Hj$W+)(5S^Xmi=lQeV6}(^XC8Od$9f6 z?#(gBk8y?{_Kp(L5WQB1p>B`Kbr2`~g-K|S{Jt=RbS?B5XMKu>Grd&OB_a+5MG(Jc z6p;_I+h+BBIQ_gIZq|cq9<8Q|m_uPIgbesP1$>!LtPdEb#Z5NeNlEhsk^_?aVDpZf zbNC(wLW#|hhH#@_f_H`SB0kqwv#cBT88?dJifuT|oLsgtAkAuf9@k09vIj*)B`2jq zMxbc(EBt-J6r|9n+y9XI6&T?1F%FUP&C4hxC&r ztjtF`io6bY`GqAVJgHH%B2{ zQ|I+^zD>>>jtUFEwCRmx3#x0shvz>fmBI6ipqVhJ3ycd0*wA(7rfJj8zzwHguJ>Tb z1O>yzJ>GAg671l1`^^jDQ`+9{A^sY=g`f3o^$cHImFptFQ>A!^*`Dw|dNlVx2dTzl z%_TVnkReG{VDzJQYX|z_9f6$B;XYi_fW?WH+F;9gI7&vp!Xy3_ti#Z3qiNLms*vol z)0&L>rJO(MHeD@C&MA3p2kK4s`lmSN8e-yBjgtu~(gbe){_hg(ShS1^gf+T-%`i7K z85+&3E$y^-V)Mn0?PkbatGLsE@=2N%rl+~NIsV%6{FUeHCx0fVKkw|CB~nlW>S>FE!ec)t7yH8=yTC;BF5% zKm51VQeK@iTvzO>fg=*583IoGjp-%H+Mk*#nxcOQ2J^>1P`AX!0JA&qZf&+Cu?KbjAE+KjPYtP~ zMWhsN$hoq&&)nzi?u$#Ka_+ZzIeD1GMA6n!CeRac?BmWi@9%WC6mNJ_-v|YRicBip zHre!>8=ydg2MiclG6Zt<%KSF|5O`Bmx>aEd;bOxQuOJWJS)?KTQ2gQ<>@D;q5=Trv zFqr~{X+d1fM01IAob$k;NJG`>Mq3xpxf=huu6{csrcLRW&`@ycZBMRN?ngyulxOQA zNpFlo$c50p2D?YrTxHzJOOA(ImIb3q)Nubv4NG{Y@b!))WA^0qj7D#`WHBIDsYLeM z^0S@XfeiLqC`+q~0oTRV%Nej(By8+i_!P0Vm2!#f?L||OQM;Uu#ec~e7oL|GgfSln zQ0qR1%_t4U+Atm7RBGIW}nW<5`Ls{%OFHJ!stQl3lBXzxeey?UUs?G^nv6(5|0bkS# zB0~1WaFp(;#R!>clwbML@erR!_w^T|2$TW@q%t+P@|a*LUx;_(Y-yDevg?N>|TdQ5!ShEd%??; zXyBm5CIcKLF#F-?HYZ4NJ9%|@S@eUSrs3)5nZ6b&#;8hUxOm6*EjK&J&ioyf>TN-( zZP|xs(ur|-?NM@md&utS&`CmK#C*MS3-NmH0Eho?7(jmDlZNivZ<2J3 zdnEdbZgzL4#zk>%@&fp@X&Eb%YIGKxVvfPQ{%O!VYnJ+sI^hOE$ruGouuh8KU6e5p8Tg&eG7e@JR#Xvg#Nvc@nzcEPo)WGq z?y>4bNOM^c8z*DZD0dMGU?tqe-FlC+3R3&fS<7$uCU#3tZs&3~CYGhr&>*+DdC(SR z+QISWi0`6yt^`)(){3A> zTG-2M(L_iljbnxB^lg-4sz((|#ltpO@ZV=F)Y=rM5e{QST$;slgViEfE864vQQrqB z6qY!>Bnhpm8eh%z)7MxvcRK&Chn6p-%@n^+QoBNhJHF>q#1`9 zfiTPaR}5M&w|H9sTNeNg!^#7z+2jOB>xw^~%B_Dqd_4Y*|Lswti)?bjRfE3N`&q{K zi3_zO=arTn?p#3=9#-Qt_=&mX)b+(aueg()LW`w~c{gws%usMf+#1tQKMGy0u)3BReHCQ1?O4GC=*i}C~t#%<;3N?e8E z$PpLOoV&vBc7|0Ld5u&0z=Kd{P`CLufA6^IjeOnjcy6Nm5{OG;l%mfa6~qDfGy51l z0w{v_@7BQpebJ{JBLF~*i3uKv&4j4WYd)=`Ia>FId_?x9Q7j&M}` zX--dCN0n$cOopwjtgK`4I##tVK8(T}&Z$9JptdvNYHGgzZ<;FJt%=;{ebkMo^$5Ua zxA0_iAj2s@{yVT6TQvcI91Mp1^2dlDue40NPon7y8s;+}UNLd)NZHFK^D8b7jbR$oB>hO-jyyB{XYknMCwVmj`Sdd(h>ED`)Y zMuqu9xY)~Ymu*gnHJPAdYhBF~Bi&oxmTgE`JCV$QmqDOOkZ7p1e7amJz9=mxp;fff z;p7;48Y9CGZUH`_ht>XpRdZ$A>Ac$du>ZFs8%VU>`PiM8wdksPYucq`o9mGWbsd)4 z<#ys&u?^ljG(UT$nr^#hVh3GfR1S((7DvrTEm@BiEOzksU%W|k!kWkP(y}dfNx_W7 z<~H+A5W>NRCJznB?kW30%<;2Q8EIe2z3Y96ZyF^eES#jBJ<};`BS)&vj_CF0NPVyta zdq3_hZr=%Jcl~{wH$FXq(M{RGxV+1&L(aaV+BA4{CSDEbwGB>L$=d5#?6*5?M6Ufn zm@#&G$6d)OU<9^#y)*hg(UFFmaADqZIb4bf&fN8C1nK)6prFG4#eDbp+(yMyh*4CS z`hB@9&2R9HX@_HcOjqbh&W2r~12~**6b-x}l?!6^(uOj1R$l}?KV65>V=0ku8w3V> zI?>7(!VTUQia(5eVeVh-u&tp4Tf{(~&tij?m(#)PHg+qBeSrgkN6of8OAnq{*1&r* z(*OhCQ}t}Mz(ys@<%I5&6XcZT#B{!H=y=fV+`d8p+`ZQ@Z~LditukJ3Ubjn` z9_4Vr^d0bxuoW$U3X?dX`Q53UCm6GI=;QxmP}8?Sx>lL@Ayoj8eKZSHW()wZ>2Q$0f>Vnf>rM3TUvkeV{QC7_s8C71-6TbR(@ z@u#2a;03{%TRTHL%Zu9-sW${k^ZH?jwA&_Hv3dSzci%zxz_?JiJC17??R4I-`{Ph* zt_DIKTG@0TZ(*SLDeIWSQ-u*;5|NPjVB>*8O$eGsbdUQpksy&FAyj7!Y1C5=L$~o% z8-7;5dDIK{zDWDM3*NQ-0oFRBWZxZv&~#ElccEf!gS(_nzqqDWxq%F4m$r%YMaIy| ze#A+L>HE#6lbo*d61ED)O2f)^R_x@4-uS+$LA%|f9hTUpyzbEYpJ|NHRCX{LnQc>q zFjbSDD9LszwCv(w&ynby`E_Y^?zSlL#X4WOv&-{tGd`xV$Su!Ir7QzvUNjrtBR9(M(&@OiwE^8<&23=( zJygYxOrPrJVw2jzEi##wjBLf9^wywYa3aV7Zr;=TO3tJEm0iAg!C{s&7Ux$3YYnEx z61GXpc&^({n(IEwE0~3j2pHzJDTcxA_o4MjU0plhWS_PP4Oh18i=dfs(=YDF>Fy^x z>$D`Ttk=*?J!uNOVhe7*pM`$(1hFKjDz5K6j@-H#mlx?+di*^}r?qOsS^k<^>ju-*Q%QnH&ZyV_>SzQ&Rsw>A>Q&Wu)Tgsk;l_bu1zm>Wq3r65&A zpI-xE-tlNVsJqwn~Zq8xA4faP{*ez%weI51+Wy#Vg^&Vc3JVP!7B zaa0l75bC_FrXYJ%(xeu3xckYAKNR%-JWSx=1$gOZJWiVF&fTS+s)L3QdjIsBUn0}? zxb>OR*FCwSzp15%pM>HB06uH^_7M2(jB`$uP8C4+0XRDsN(B(`@nQ1b0lGZ+>yjp# z_i-6tNpF%hMR5KcVHMxI{hZCDv zec~en;Hc|9+W)K-`B%sOg3~CKX(@U(RTZupqQqj7v(}|%-x~#17}RmK&pc&3QE_Ld zZ=U5h<-YcD^XGGp6Q;o#cHlTG>ux~vh--C&F7t1Ud}!)&91rckd@vk-8Ho;3fdhJ@ z6ZgMP%Xrz(biW$uvcjKUR+lh^J{xP0N-dL4Zi}%FA#yX-!%{J{@n+Kyk2I3p?JRM5 z=XZ;<^Tw}Y^4r+dW&{JGkXcB=?+If!og`F79mC z_erB*)A9}Be5<5?^0ahITpAb4mlyet{H6@oNxTHrdiz2zE4T(k`0pa$0($>MOh5VO zJ*$m@?3w}aa93{q^Z2#^TjP)GV*u-R`wg}ZmORN3;Xk6%*HY#t(5-%x9)Gk>axatk zy0o6|&v7hK_r6|IV9A)`g2Vt`oMsSsf8>cW@O6|;IaS+Oe`JiisLwF#a1cTL-lnYM zT@SlpK$+a6+Hzkn3nB~1+s@}(w9WNdn_f`p_yS!a4vRHub-VXj%XK#MwJeXNp^lxd z2)xbKA;_Z>*gOKra_wo~0^;DS-gfixcwhhdGrRTyo6}T%CB_CcW&J0X#~iuxnRmLl zNN`lU94vkIev$7EFrV17r62wsrxg6a|IWR0!^&(ScH z^8zNG)N%yq7y68Qf&i>ij&Zo@!NWz4V~HSij{#&XHYemoi_+anPK5hlv=j>EX#@V1 zWG6Xp%vlQut-l*jW3*D@q9tj$^YSE4E*`H(3gw!QM;U;j%gRJ3{)WVHl zKgFbV+IeN8ky~;UC2~>dW$dk~&d@Ia>^)Y5! zW<||Cj``oH`==mF+OQ21t?Dk@wq0H7vTfV8ZQHhOtIM|C)n(hp+UxuOiHSWiCo?BI zVxMH5W<;*cmG66B&wWjMrsyOc&~eY2Tgo|U$$&#CXC*wW&8t6XJkUjN#E^r>mMU5O z{xY0}wTSD0*ip1~RZbBUCU!8)v}0P4UWl(}WT4kkYvA0Y0^K^wpg{$SY($w3VD%a@ zN#gk1{xvSV*@MkYTQpP6TLHMMHUJQF1n2%+6U|Z)YRXjOk^5KI$!<$THL~%#;0kQ6 zvk^XTj0Rw{-YTYS+ze4XICHQpmu?46s#V2dX!}-|i8Qo|V*l>nI()pVf9M~R{jNiH z+cEjLwK)U6U}(iNG4LB?0l0X0s!x5z;qQI!4=S&HS_=^I@-y$HuBV8FMqAb4{y)Eq zFIRF#Pr%{pE(Ol0KH_GJIE*|F^dz75-@8AH5aN8=9ZVdw7gvg=a}D1ZQYPSNd{Z(J znU%}6n35EWEb7%tmd|@17~qMC^}}5z^pY@45LsumE=1Zc+#(Z&<-X2C#Cvr1n-v3` zjf@eGtf!JI)v}C6BosCY-JJE8%Ml?~{g=#fPxw<;&g`lIE9d-#Jlr1lNC0=rcte1$ z*-Gs)af*F)YhLw!;2)kpr6~i!Tnn%95*+>4jA$v>_7mPukcL@_*a(E2o_FM~vt3 zi4`u8m}|=Xy!B{!orDqiXGgLUVuvd$inrXJE5qCNw!?>azH_}H0q8RGLTvQEifJ|1~4=9$Dfw-`42WjNCbO8Cl}WEE=4FK<(5p?sbaiEXeP6Sx^ix_;*Ef z&J~p)1b$zwA{CU71Z&yYQ)ZcVGwS9!)b|zHq*0vMJYXfGQ}WIUX=$`tPwU$m#sYYw zrP?_FU)0g&3#LiOHmyurvw(rU1{}cM;aPJz;t4>Phx;%E1AGcqE-GcWDk+;qo4KqEjcLmKCxt%eaimr(E*Dna`4(#+Be(>aF=u3P$Xg!Z{PVyw@pyWZnDPvGorm_6CuA{B*Kg^H z3*i8g*F64j_3dy#6=7Mr?;YV2WuM%;DX?Q*XNHA2(rC12jIVE^d+a+y&olnZD9tk- zu(?U>U+kGO%jx-W&i3Mi-MIUvIBr3>d346_ACI?-@d3Dex~nGC`)GDpVbXdAEG(Hr zrhfT5JRTAQhH`nf5ja<1Ww%R-N$XQ3PO$Bl&yyvylBs8gaI;txaQ%4FDadVxH6=!2 z?_amF0kU-(c?}si$=N(KFlh-?#Qu_wL*{p%_b5V>x7H|7(6m&GIUjkDJ`hleF-tAG zqH-*970?%uu`4XF);u46jb8i6zp45PmCCI)pZ?fiaU0NT4#ESqjTe)}OFq*ZV(5G)r>~&M^C; zOOJ)1n-&w2^_0X_$gP*XzH;kzzGB>mkpRhY1^F-eJa@hMZlvby!?PHjsp{ME?|#C| zpH+-t1zFX4TID}H;SJ~PD{1V9lrr$3V3HaE~Pr=?iH zQZoUNbHnhjI{9sR%~=W|Fr5--=F&BpX(&lB0~+j;4jW3xL+xm)0+=WbUmd^o=uni` zCdX{8>YZqbgZ*uzm%dD*emsylNw4GtgV|^oF~0RYNOG@SNGZk{4Y_Dm_wV0vOOF0x z#U+4AvJ}1<5`T>Ab<9KEq~k;3%)8e$yw2&d%`Hl78@E9ezB$8O^u=iEXeXyah7`te zG9$-vX!syjOz+=%5ywbQlXhgh?;X&+sE>XD+M*bh=? z0@wkt(W)iVe?4hr^(u_r8UsA@%+OS0kl*-@Mm`=zGHV+)bMdh|ATGBb0w{ApkVODW z-iR3}Z!C))JUK;&O@X>1l8NQM>z#ywu9sk@{yj#Mw>4LLd5S!ec)$5P^HFB88XSCS zB7>P`=CzSD+{R&WM(D`K!C`vk>qrUE$r*Q!4u{yZc)@!8wjVx5^S68jP2PBrfo}Pl z2YE*}zA9%!K3V$)FowPvKU6eD>f>WifW!$|L8K19)57vd=rbCKx+exm!#E)$O^@E| z;T-Isrs8RP>n|~~6xad(RtHbikswi1$3H|V5%RZuf1&=V8B0$#cqRnRzigLtuV^9fYOtVO>8haQ`I!P<0}eHgkI5jgO0&{h?r|m|LJ&5(ZQO~V<`CmhlS3E zR2J%Nl(x$!8%xD6H`MjwEH0GNgD`E1c{D9qBKEMfq+n3y6Qqhr=VgOm7-0BSn;Ey) zYXY8xZnFUcPF3Pa3BZHpebN7#Wd9;Dui-EL0qy(bs}H(e%ms!f56Cot=s@oTc&j|) z%Qt`L>oaNM9&VYY_?-Wg)*;`LzOrlv@KzuBmc(`XN8mjBicQ(GB$+6!d#lH*KiTp`~0TT+J&TDU$lj3A?NpF+~wPccaTn3)V zvLpjtNA=fkqp<|fw)4)2OPn42DLC@-7?f$*#Muv(21qNr5Lu@0bAkMikb2`D3F6pM zxFl<2_<9LW$o2}4f%up?AGww}oP zp9!9R{fIZyq>Ia;lI^}g+5YLTV6Iht<1d_}kvy7q1K8WUwPgD?_i%YM0Q4oBhj%CG z^(#$g3-n@n6Ix994?~j!IKKC6?URmKw9x3lx;iaxqrjPFush2<$qwl|6tGy56lXOh zib^CD!4|(sg-qv%gy+Lix;$rzA_KK>_JKq80?{5u*CV-Ke7< z9QNUKYZX$KYFC2G#aP9A8NA$FTwYu}UUx)BbA)m|{C!HGROC_O;=L~Dl zXdOC6|2BzPHL2vLsvp2-j<{enC8#hH9Fu}bM9f%)JN#va*Mn@R+@i#?=KtYPWkyeA zLV>Jk6cU?2&st2EFs=ORgjU7w9VXt98Mh(8RK#FmLzLJWMo|FMm=$WmoE2ok+!|w@ z0t;~R$^fhEVmt}lsczf@3E&l9rC5@6L_-kj!aPW_Xd6DkB-!nDS%B28!I)leHa*R<#%9YqaZy^9s0Ot2WlOp^Z= zf-boJ4P%XfC-wWX-j|EF!EyjO5-~7w{_HqI?IloaV2H}}R@ZgXA zOj7J@%b!cfYS?tPF%v`6L{lv3WKS%TQ=<$Rpg$Y|zjID?f^d)pLty8GIeGTD?Dsr= z?&xn`O{U#^%&6>3>v;FwGL80r0F^Pu@;|t^qVgmm40f`6E4nprc%)(L@+x_W8glomSAuJRDY=AL=SWVu7F+6WBR z0!icE!~F0&mgHd7ugwvtvoJyjteuC*8Pue4pYDVZ2Z|`aP(WKQl_99Xk$o(v%PTVA zQ9{TF=yUUT3zV2K51uAy>8G;`1rHz8=azDYMf<#7wq%M8tm2DHumV%*jzr(YZf2I1 zTYL3+IK4o5dovR{L8J%@7d^;nJ$GgNxV&K@RjD~DOUdtzl%ZXF=JMIvU*FN;Nxo0! zf|yKmATBS|^YtMo)&IpW1&kKYh&FQ9?=TjDERl5}v3^1}^Al(bpMs)qerR(|JOiuPE@mzcY4H!cOz-r_hqg=! z7H4jdGk`QfS(4_AGkIycAPJ2oNP)ici++qEFV*1CgQ}uYVlGy%VLb?olBOZ^s3y39 ziZk|<_HKBUjIp3&smpP+NtlG>Z!pRVGd-SS5ff&h{Ap4H=2aL4btbRf_(WMSDtio# zvqZ|!W-|K&Nwrwyitos{5P@5$8A^e#7*-J4;|@+=fjJX%KOs@bQ+PLPjYz99pY}?= zuu8YwU-rR|PyRgvIZgeV>*|x#S3sm3A1Dk5(UUfOiOjd+<)#d?W6KQ_a)g%#-aZ7O z)dhY?2v6KZF?AV>B8OHfvoxoFZOei{Uvk3^KzkoBD=J02LdRJt|-oU23~(He5B zL~>w}yQ+v{uFv;gLWm+O_krMr`P2x1RClB`S5mY-R1_W!;#lFvTVlaVCo`n4p6ade zW+kUgJ23@H+!s#IIVs1=?GJ&n@amC!XvaFdnv5^AjR|O>BDCB{&|7RB7MPhG;rD?q zP84B2&@Mg{h6eH}1z5~ism^hl5pB(6*m zNfo-~1cR;=>gv|y{9ZXKay#Pz7%xtewV(c{{AUwxkVoF0r2Hg=T&eNg0TDzq7(s}) z>{H@-qCy8ohKzBeoHLj`cR7rs4r4@u0T-Et<>(ltaw3w%YGM|-$yvM_($C236{8X@ zb<`0Mo|B%D{-qeJma1+)_WlMbZ1vh$>M>wp7GwtJl@%I@DPhwcGw+}ffTw^tav`qd zA^qbvXaCVaXTCQ7W{LB>UEXsu@P$#DM{x1SWN_z!QE|Wo306REide6Iyp})xDZmu z?>h;+*C6njR@wqXEVoyjqlKdu2RKSIMZFGA_P&5vqtrrR15eQFP9A96ITlvOwd? zHHI+o8{kh$5Rd%3VD9{-&vgA|j5?Yn&AD$}?GKDyC&1UsMIjd8#pO`}aATiC;7pD~ zN1CKU4?x)F@k?3kAJf_9bt78PBR9{;)gJUTujHS)r=q4Pd4Ruhw_6F6dN%+5Z`B;1 z^s8s@5Knk%*yKH)_o=$=7fhP$Py=h$pl=07NIx{VX^O8rYakWKTk`-vI4Xv@R}PFV zk3yL|fs7_c{EDl{WadWkO9x{=D>9t@!r9Cz{@rHsWa}9PsS0nyXZ&INJ|wUrxd1D1 zw(C*n2a%teS|svW_qVCsL0oJt#IF?7O)cYYTOLXC%!D~^f1LNB@*ctP^~xXLTHC8{ z{*f@QwUqDJ%|U8T(W!j=Tq~&-oh^ypD}bQ(l;;$Sw2|mOu4}}pNZ!S@Rg=dgfj?4D zFxRS65c@rvxW#|XS?XLcdmSIXE3oY;fb#Hoqf6{5#=WRpig*~aA9~M>`0Qy{38M(A zPIAQuIC-*%R`E!rltT3~pz6$_f4|CX!r(WxG(!scxsvy&ZRMM^L)lHMFN0nec5-+)dw%I8-`z|O8&6Xd1q!6XkRRm`mgN~uxxaBscTrTK-_|ELn?7iFtyE&bx@ty zJ?G=I0mMTb`VM*bGtkBPV4hYBnC~yd4jpsa22G^~Ea?%A?ghdL?#OX1hI6=Ae(3+CPM^TikRL5C)ksgQvn?0f#y?4v zYo}x^IK5ALuncG=O)fw*@Y#hA$~uI*m{^U>O6g}}<8zxJf?QyE;&X4M{$Wz$bCW2S z(eK1T5@Q~pJs+wq7Onj!Jqo-;+aty^vCNnf0it)JBts9|2 z#S(wfwh`eU;kDMvsrp~>jEjY50e+|T&wOtkd=;)0ga+aQQX5q_y z3X6Ot_0Olh-VB+CuYB|s7YB#G@3`NC@LFvSCv~CvN4BOH__|2mijluOjCNA1{eDXg z-LA=Ll64!=YEsWUHabf+A7ybaaubT#f?J%t{{3g6;1 zUrl#-0{XzMHF&F=0yMAYZ1*}tJAs|ZC6V_|Z)cbECEusg`VDbRort;8YA$|ue|QA$ z8+<`h@n;x3Horroptg<{JJPPcJghRPg3r7m#RgqGD7wmtISoE2T*Bn$e!X-Q8>`7w4aKD2IUM&ToNuVpv z)Wy2R2yd@(Wkv1h?d)a5&gJ`1^+}vhIrd38YeM|n5hH&5I`j33L-SItnP_?2-s4|o zK-lE@B$GB@-LQ#-ntHU%Yxmz1{pA7m%CqMU8g2HuCajn+*0lTZ$aKQvZjgVM`}MoU z+GcB6q7xBtE>v?demIH6Lqn9|1jMLu=F@3^rm*4tW%j|z9;1!I81#?+=$GI5sIBI% zk3f~z)|Tr;6KsHIA6HZDsPcnAFB;MK6=;C(dA}vYG?$G$qA8^hou{@Z-eAGHPMgx4 zD2$R;vLcubvuw0-gi1LLrZ zz2*Vo?|N>}YjJ?GssK(ejqcwW1e4ALW}KZC4^`_(ES0_y;bq5*K3~un)7$%RB%ajKGSqH(FJ&<%8NxR6KYH$$Uzl4b~A)n-~OMPZdpFJrU-M7+K>yZLbaQPjkGeeu{C7%Q{_deVrREag*Yr7t~G z!*-bX=LV|uy;y5EmC8j)=jKbx_(!i~{lUYd;+`9S9)G~yjOCgaWE@S8CqbW2i@VqJ z0pNRVEPr;}t@ay#nPEonH!cHrYP~lrob$p>y!lPnlmNy~fXH7|@xiG-rq^%DE~7`Od4+D^sSk^myUv#0FXZje=~fE| z>WZsfVZYlU>Stt(5#T7&^X94awn>&oY6WXfQE{@c^m{IwNlu*!f(sZ~$kPFjED~i_ z{_Z0f^!ZP986Qglb-yea3wRf%&kGH=Q}Z>PZ0%eRgPihlKYtcc^-e8iW*!C^_)+Ul z)ZKHNQSX7%AO90~9u?C8HAJl`fLV{|z@ql}V9f8iR=KP7nw#PYT9916RM^tYDUk$j zcT?!~>pU>XhG^O2kG_+yJB)<`7547Q(mrwYuF^0x&hInC9kNfg1J zaxy6RD4{-_l)(Tid9JgVGnJmXiFY6>n>KKh!^B1N3t^x4 z0|xDePwjt3aPfS=`#Yg@HgR7FLA2eSB-<2cI{CBeW5r>GGlf-S&`s3iuNfT{ z>_-lG%Xh->9g4yel>&jYUAhoRmRA%En`NMVwYt8GqE|a_h|3~ZP@>FXBjJ{08GkI$ zHfG2WUr7`h8;!q-pZmUl?~!a4JbtZs27kLRE-V>u#P6-7uV zditU!s`iVeLZf9SucxH`em@cF5h~iQ=+~Qa`p8cvKE_5T>ynS-+wXFh<-C>HEnf=% zMQIya(yRlI!tU7}=&NA+{-_%s&TsWSBnH`^FPECF-#dFtkZ(73dESYDln?!xVta2o zZ`eF9dY)`nj&s8-%9+Kr=PYXiH&{96ncPi2=V5CmNBYhS*PTbZ4RARWf$E<*#ap^v zeZ)|;v-VGB^Z|8yoyRFc{`GOz&$gcw?ZEnDjC!`_g?B#d%-P4*%(1NGveg&H3cvOs zcfsuYyDp=@TJv(wrE8MslW>24?Z-0=Zen$^Q$uoQugC!%>6mIbmd znY0_t$TR#B{S2^gxyX}U7s$(Eo1X{t>4uYwvl^6;zs4Xs1*|x!ra!8h}Y~^ z5sG&ksFjYRe?F*Vu*iU%JB3vxHIPkIj>@kQrKg6*ho5ZcZ;t!-<691RH4S?+jt`lt z^<%u*b8u8m`isjW5Z?0xVFgme%Bi5s5jtXT$1J@WmPJiB-gRR91HD?(xOlnpP4b@! z$nFB(K47k_weL8uKa%qHr{t0k*V;{c5wdP$CO$&p^1=rhNjGRBz5rG3kV$ixxfgr-dNX%?{k_>Nmv$UdCx5=Hv_ z+B88VGV~Ppi%^&FLaH3vhgyIHJ9yA%`sGvogm5mJ)f^ZV2R2dvKg_G4;{J1j8|Uws zY#aN_)OU~hF`)wF9!?j%KX%TNRp0TE000RrR6N08d7!RK}2jG5xxg6!~z1>&^T{X)- z*_hS${uWFV_W3{%LKXA=^*?cQL2o6_V3i`KwZ5~O1M}SFe@<|%^#5^!hnOW=EtIMG zaLPip0-fM@*WTN>DdL=O3c2e_2ILEvoW@^MmW-%y_4OS3eScLkr(HRV!K;TLn&E-E z5JBMDbwJP_r~^*}!%QCbwycoWlCk!QTM#Bc+hkiE28G0kf_Pl(JhFj&<;qf!e4NzOXak|-!y?( z1UwIrz)X0xpd%|Ddt2OELrYBcbrRR^dVPK2+a)|iE`RFwHX~GY*bj6;Y(_mtxb0ryiPR)Fz1&9PenCJnPcGqNP%_5Dvp$!TT}YdqNdSRn@RfV|AJ& zzi1#b7EI(0JOqO>7DS-nypnVU#T5fhyN%1xUT;LrQDP;i4tRQupvjI6&B_qK8G*U3 z$hAqHFuLyP&v=<#*2%ZHALMRkm`T! zKoYK!;S3QLLxkDxsh9qV*+|C_^nhNMC4hcuv7|v44|`^pMiTKGz)x9pN-Ugr$7>|w z62*~Uag4*Vh>%Ffzz9An+rICc%nsR+LI|6lQ?=C{iue;fM!} zh^nt@WZ3EeBtS91SNyZoj<%cePz7F~H$goH(i05y zISbo2?M4$)Na;)X#1S{fGt|_RSLg zru)~}1Ec#srQ|x2FtlT7uegJ{ZICH#v zX)NQKp1g*`L@na@KF&AU9Cy@-6nESY<&+izXXkUohA9CQgD>oAxq?p8(n7^99ipId zacl@Q{KZ^U>kkRQaug;I#R)-7%sHeEc}+3Cx z@&mThZO)Rll4vnV6_IkMFRZ$$`}`%EMs>F>YY zvKC15axVO1#mgti1^q@RrTerfL~9?}Q>I)wGdHtYb^1038bXqSsRvCunk=YV)eHvn zF^i>zMq+iUG!gXWD3k|N@L%_Q8v;{+QV}pcCJQZExd0Q*&_W>{sFvjpKiphP-1Vdh zMN_xI#461J&v}v_t5>7J4Dd-0g7YbNppz!0HSl6lj4E4q`)I`{k4<*vhD#71fg_cl zU3@n>jLhnQ7f8y*A)V#CzBms^sBRmMZhEK6MuuIvT|^j0r);5znObO->seQwXFqw0 z#8M?>+CzecZVk1@*soo)EpSH5>`8b}X=%Qruhkv~hn8PbV7|)%w|LZ|LQ!MRuLU)dgqkD*%B_E_HlqVWwli09YFkD*d zq#pDVMIRv*4BMikHII3LZ)wEbp7r!+v%V*#Ue=f>DI+baTfUe}zlZe*r=O}*@GW$;pE`%sQZk>5A=L-9G&%+R=zW28R=)r$l z@c9Rv5{1g6&Jr{3v5Gdj$Vw^D(OEMZFn81b_#}$UZyJF~lVaIl=Otw?P)3=yEy&4_ z?19qUfn^|VN&i{FO;Uy=q=;7>uO^qFQsB8qDNicW!fa5;h0GXnyGpWh7LaZx4N)t) z;4{gUVE=~w?FA(f`JfOZa%;nzaW|?_x)dO~xKW0{WPs3CQcFm+1%bDjRQ-w7k@{U3 zB?;|2B1iGYj2*nwtHnPd6p$~8QBJSWgnLdP(fKGjiOJ;FV9g=sdtCe#_np3^0z3hzOyjb3Ap#$ zVVL1y1pwzB>zbazp)|h zK|Q0Vt6AvD!5kfQ8bf>KG8SZVoQ+g&QHHOd>5I9hQK?|cO#icjA07&)oo3g2Vx;*o zpJ(CshD`nRH7zKd!#0G*OM@%A9pZ!nE4s)qGq_D%8q^8d@`dL2tGEBxL>D&?l5|Ww zvmI?jFp9NP%7csh!r?=9f?q@NrD?O-&7dX_~C2b~et5X3>F zJBjz*_dJP7C8#fDdBO0}o0|R8f#wm2nD=w{hvH%~G3Hwz%90)R6yL@N=~ZYRhn^FgC$rS~||i{EJL7ZEN*nGwy< z9h*AU!vtMBy=20G-EqQ4h(mdkQ-16y|7$LO*|!J*J>LHT#k6jPJKFK6A<5mjNxvZf zhZAca=;TkKI43y1t_T%UK}nmZTk=QvEabl(T--AYc_~BML()PF9IpSAPhIe-FU_4DT|ap-#>qGc>Ok<3;%Me$!#Cn-rOrZS?Ok3z*oOtCrR7T zX349(OjL@cTT(?0%8@oYl`(6?`w#YN?E=DH&M$w%+Yhe(B}43A*%SnW{By4k?@w^A zitdlNSq!>MY!~)2>V4ypbQs*7r~?vGT1hnMoYXYJ_sMb)I~Q2I?!-%RiUK!OEeQ(lT$Np z+`+-YgGhh;K(&=L>GdhU^-HC zp!Va(LE?a#?2g%~uBb-pZ{oSZe*?{wj52;|50s&LWES0a%fFptDzn|@-0VhTB2bQ) z(;sIsa&_qc(}Lr{X%wdg8**RQ{rX?PDZMuc$y>~s>)=});o8j04nllES#s|fEb`wZ zapVM^th?4p5b-mEGuJU!8)}pBe~eRrEA4gLpK@1SyX-}B2Ir2@0YBezZEnHs|ARsb zo2zabR=;j5d9a?0zL&KueNy=`fGHoRClGH>VUi~i9wX7@?J72r`RVf9W)!3w!JdgLTW$i%;=F|0 z?4e+!zm%0z#X)-dQi~B;#sR(Hs_mLx4>LKcmg9bz(J#lPRR+2J|9QdVk||Q7SJ8cy z#$W|iuYg|gsxj`M0}Z_?;)&^V>NITiS&rXOj-J1$Lk+`om~fmpv5Aby=Yx|)Qc~6! z-%%9~C?3eDN|4Mw-X-OKl$Yycw0eF>7V5;Q+#TR~!WX5!Z|uErlC(Pw+l<9ME<`0U zAD;8w1PB^NkIyeFZH#=|+-91 z`J8YbZARmj22)`+2)Q6O?5kiZy-54+dQ4{Sf5-PA@{Ee`H{~a5$yTW*_J883MXY*o zc9tg#j5<$(9T>MvB37tTfbSK}Cz~_S>1#4h!aG+7YQc@?p7($J2hB9l!X5qsXl2-YS`m4H?VXm4frO`hfq^eD1 z{+TK8WcK+q=gj|wWN6ZV;{9JB8Ki;iX6w^WR(S85rR|sf&xGHG)D^KO;2x$^80D4` zsBe5dhHG>?0QBx{g>rD-_^>rrXfWZCaXX5i2ft1tr#32&OEJm2kz{b40xjU)rL2qQ z(-AzxX0j9$v{UxBGO~|E)ZVA5DR~O-Dbp>^OxK$^jtqYx#SOoOQ)hy~GAj+Hgc%Yq z9qo^K6t3$M=*q#4IT&Fag=Q}iF}y&s5+(+o1b(j;wNg>@l_;DA$(OsWbrOSX2bv~}B;?{s^}4Vy)P&#P29OWi zKWqE|+QID|jA{`+D&2U#Os$aOGx!V(<(ZKx4Vz3WnG~lOkCVL?pOO(caeMmhVv`BI=t*`CLFI0;S`??-|c@C=He>?q0S zf-oMMwX*COg{S^|{-m8K!gZ6uAR`g3n?+~nfLBdTaH^1}YFE$pW%G>lm2L!W&jp?_9UflWbzzMEFc@1GzVRSiRnV%Mr`XW`S- z^p%EXe>_U+Yir-p#GoN{y)7S=k}EIdny}UX!0vQ-`Y*?@qgsL-h%-IM9Hl#mdL)a3 zvw$*wq{rb-xk(s@JK33yvGHV&yRImo4~PncQZHY#o3hxfuTGM{eJS#?hb{aSB~~4m zcV(Jo^`xc6j0{_Zg_9=DEKaD`pSe&q|3)bGD_^7+D>(pum>e}+W z5fon~A?v)|*|n}=qp9n9SAUu=f*ZQY3Q!D=qn_%NdwnpZnbXWb_8g$Af%nERzoszL z3`q{B%V9&!;U%q82bQCVqe4h9t}wSfh`l&Wy4QUtox_vm+&#c%O3-fEuC5kA7bCrM zj99%kWQzEdoqMc@XDhp#Z95gq%kVXsQJQS}+ps-$;eA+u3rA`WYaYwPG)}K*C)rAo ziywQ*f+T44{voxfm`Wc?v7a(G2~TRWa_u%HsnNPZe3RfJad~|^1cgOQP9MlKX|x4Z ze$@TcAZ!=jqv#OkVwSfxFRh=iDQ|lw09k!|O$;-f7`h zgd}RVH*+zcXZT85q(!bcc2^^Ac4q~B;;uHkwx+ON=e2A+aM7Ciwohbf;&-aU&jkY| zw#G+v!~g8ArVB5s%cGndHPz_wwm2I0ljUkP!!J2|T@loI)x7FJ=k=R{vp{cD5am}= z`Ich{UrYog2IBfZ7F(OV(~}G>RrBqVV-9M%@Ka_A7fiM$1FWw%;X6xs$DjN)cR zr-;~bUC+(U$2?!#WXmp9X3R(ObRSsUzpftMIQME=u54HnhBz&&n;Vzz)NtLRMB9#=mYu6=nR50vJJDP8A+@Mo~I8`n7JVQo3>R`LP(9!GM>z83W(`X z$4S9{=BxjHLk?QSeJ#<{jBiCghLtQ5xwN}i@j--67n`c*p(VHV4l!HMzjw*ILw;+B z9!YldZ`zA1EaoiT}}uyC2VXCe&zzY{PZvV$0~bM({Xzktp9Dd|)4=Jq51z5cAm zGX90nlGAeWv+ELcFn3aOD9M`a(;744N`S>t5?YwyNyy9nTT3To9!Vh z?I>FhO`jmGlUE{gw^V}Kov)695sN*C(QsM)N5CvatALspgKJ?{$fm2vR2g#Dw7uc{ zB5?uF#2m!tG1NbqR!C6!>Yv{*U2ju4_^T2~1ONPE!h*M?J3>Td+J*9VCjM2CI}i{5?1TTRr>9iV{>~q8P*?yU!~-Kh;JmUk|2yFfGlR)R zSpE4#geatk1kS>1*@$$nN+SHR+WJ26dB@XEFNko&wgE7vI0l~*FCowzk>x946Vy;Ry$cC(~Q@vw^>KY8y+9amxqK?pfpp1en; z{Zp;Dv#ievQCk4dipNbclo7uK+WtEoVtvLg{bEdQim~KxJ9rH`{US*9QFAvAMhUacnK?o*hw6#%}`jrJFm{jd?YL?b`aDRVwMINF+mNu6X0+f#n%E zKP6AgaoPWeM2wf_ptcy&8KG2u^7%_sE3 zrS+<7t^01pV)n`1l(=`fIsf;Kzt88{rS01Jj^I`5KIHLjk@DLg@u6RsU*8_jy}8SRengq3ldX;4S@7=! z0su6E-`u;HQFIbz=u2g3S@#*Iq3F#U&HA#M+YF;4{T_g+3W$w}PMj>tpV|n!{C^L^ zydWAC-@Z@M_R{hJOU>*8ppGgXWbv0hqOa6e=h**dG4;*7U<3EXTW}E4>Ymz=EmaSz zB9R5N=t07Ls!>96*9&x0Dq8H5sl7tvWg*b8{C$|H0>PJ3TxPLnf-O^Ya5nUIMP@Ln z?$OB72Mg@Ty5wpx151#BB}|7;8z1pIJ)R%$2VDn*dj9`E?8p3eA$4Y8z&lLUlC7;JZ-{?u88|Gx)fXwgxD|j!kQ9L*1d>7EK*O#~&2dboSuF z>yvkT@Q?~?+05_o_4C0h^Da=3?b^Qe@8cAa?$DDh{2W_3+Pk-_PP6?1d#9^dON&QsX6)*V=9EiW{S41) z4r^<#zu;3BdN=Ifl=vO=rPUy}EZQD9C3UlT^fEhpw?-BxTP*@N=Ar!JOfJUz7Kvb1 zg2WffQ<{*@Ai0$Q*?cF;6CKaLGZZ_qhx`+PhwZUJK(dG3M=ciM;pXu8DEQqKZQlHf z$PKT{3s;g>HcO3=)^Y-AIjeMgd!_ysBSqEuqsvlq?3K15H!LPG_OrZZOOnsu?q*Kn z{xsqL!`?l1M;dkw8jhWG(6MdXwrzCmjytw(+eyc^ZKGp5>DZ~Me%^Ox&CFW!59Uk$ zK&9?l*L9!!*rI@pSm9#4loS>J;gFk2G1J8L3`9@Y+hwt74}FJaD5)~o22e-cZ@9{( zMUov7XQ$+6E>h%%8AQ39*)D~LnIj1a-_-V7WLG{1G{Gm-7?EUF?RpO6(=&&(O#p-j6;*!xdN|xA1}x_A@!rLE(2W^Pp*CQaoh|c` z=9sMp;V&>>FZ1lboVs-Avw&bbLN|>{DoHL@)&?*Vol*B(vPt{;=pJ9>!|t4l(X4B* z#{dPCJC{9)NlOva77^}KgM?g6rC)j!z1ze9Lr8|-5nN(1Uuv-&tA;G%cR)lj+$WN& z)Hc;e2l=Yr&;>0$6H?kk=hP{K(xSFbjO6=IIR+mmIPUp~&A`JZI+^z(8X0)Pn02yg zSKiKv%x|)*Jgr_b^3Q*PmtrV@S@+Tzf5aESuBFlmAne<-;IF$6oDaVouTOkcP@BUO zpjML<>vOYHyY*u7rhV!tQk3qpt(cMQ?Fw7RWQGTowO~_)Xm}Bdm)A3|p*C%)9&x^S zud02kwOW}>$1%C+U$5kF5i7701Efo%7?Rj5Y+~nI0N?V|?){J;W!={R zz`?|LRp4Cg=!9Eam!%rkY!OyvgE?DUVOsQiZq?>w{TKn|KfEFQGsZ24s(q7^JIV^E zH#j76_?AhZUd0FN0~djJBLEkHBLf$K7hRa>pZz!9pp+i>f7lI593dZ9huOi(GyDJ+ znzz4(5I{a2F!BD^@Nw{=f$^*8>u^=N_tH`hy#Kb3c*E=RK5NSBGVUzpd5zJ0f5(bI z-}C>*8$Re10po!K_eJsZ0vxl8ex6_K-xlH@Z+x6<|5o(?W(^2?-}agX;a3SvUcUsn za0Ix^F9H3HL1Fg*h1?t+$Dgyj8tlT$uM*rE_nLx!O2hGf+d;@7%DsNr=!>sdOPuPW zTI}b>S2i+J4+uW`TQ;=qDGNo#%n=3>HmBu4_g>>&b;}ELohJ>-5SgaV&AyIMJOE(N#Li z3h|o&Hw7V{b&fMmIsY^3R##c1zZ>l{<>6tpSoU0c*~GOr2IVS-6f1K$a$@o=iZLoS zVyUtEgy(@C=Hv=`ozLvMv}I*5MY%Tx=wMvaCWNH4nvyzKPr%Wold( z!K)+>C9Lx?Eogi>%{o1d9h|y$NIa0dM#*1u=V5gJ&H~8;aeT=PSYSLjWY9th#_YI} ziq9|(jvVWl+yh+*?g7OqXo6Q-E29gsQv+k>b3gv}5g=G4?8r{Y+7-7_*A!j(H*Q$r zSP$~X2_z(BzuZ(96q>GGiCG?#Hh)DwobZ?e5q(GnZOl~+R(+dVtduT6bxI2f13_E* zmqcw{A^CyPRgcnmA)`(yt(zPVW3)eQboP-c<6Q>hWJgkz=UrM<%}F;WjHa#F_%A1l zXBVvozRHYkFVAt3fVr)s4tb8wudN&gofZ74_@BGvIY`g>wSRdDIwlh+DQ&3 z8{^OgRYmtUC85G%JXK0)_Qc;nR4U#^xpb&5!eZ&ZhQQ~PUQI#S!Q8hZO7vO=8hMiu z)Wk+@&uI}dlmJxlSm1-B=LFwbBm{>&4v0Cv)T<>PPnJHtFn=e8Y*KU@Df`ofuKvPr zEOImE!*W6=36K4rT%RvgIj$ujeUE*YPq3txh^*#;Qnwo=q5cRq6%;7aulI{sZAFSI z0dGFx=u$^2^j3tE?vA_jZ#UrC@2mEi-s9s%U#D*VV063nFs6iW;!mT9fWfIfQ#XQr z1MURu5g#H+z8oKIVu+HMtRdm(gcxkboK1j&;|N?RGHW2((6!{`@r9RqIi+dB4Eqzt z8Q-c*bCgVq2EuOmBh)aq$6$yOxwI0J!irQw_7d|@;lbI=4CeOX2@{VjWS+eyH% z0FUvBR&|ZC3g)ARk*UOye3(*!L0j0J`CCoX76{P>CH4SbF>ILip=xP+%vvfIzbE*n zJTdrNvpF4_M8rLZESj)yA5QALQ$pdg2Vp%4rx2d(AIEr1^Kh|LEcC$BGPd00PQ!7- zav}+$htqdaNYvchKQC3^bk2}8EcMIiZSwSSn4?soiY+wto>Xfc86Aau%K&l z@y`EjiIs&@-3_Pd$Nf_q3w^F01W1K7M8{cF@m;zoOt+x>Knmh%(b&*@ zZ?O7E6KOFm;$^MuD5b<`^gy!-v}&5Y2@e<{C13TZh~*JbkRp)x*caI?4R^z_3LJlL z;##b4cQ9Zxbu(mrAhHm zS!oF&m%oSWQSjL+)yM5%GQA+O(R?oxh;Jc!EXe<4p&k z9MZvJZ@x^}FvQnT9v1dDpMxc9Y9Qm3jgwQkI5Y$pH8U1h`@@1V?}ze5vcV9NaF46P z+%jUy;({uWl@t<-MK5#GQyPv9pVXpSu~~1tc#8z zJ3u~~n))jQ^`Zr--kLF0<@X8hY(qt`r2Adg8p`2engjtA=1TW`DfSJH0 zC({@|M1uLwAT3J1KlU7I1p;B-FSJkPzbujTgz0plXLLn@8j>;T5$F23iI02#IpWp46!r(h!=zr@k|svRhcH$5*Avl7e4P!h0|Tws{oT;TwaL zh-oWnNyCZ%UWJWhY9^Bi)XZ{28gH#7?Y~!pqpn$FV3Og6=ekIXHLO-+0C{DA#C}oO z(@K@l=zB0P{4QUA^<={rckZE|KlLxw;HWtZ5g4 zY4)hhL5fkiSAri-t6-`7J+;Ut$Fu%-{?C~UH0DYH;{jp}RBQNEx*^@#4WSb%7BAvQ zYAcHaU7gl&cvRpJu<1TG{M>1?5?PfgpKD7H0aFn?u@2S~qzuKqF_MfxC2v-30z5ns zXugiLx-q#Hh%1|!RU3T=6qHAl>6Ch|XVyUP2GNutlsXqY@(!UVN3L-Fm=F$*Go zW!@2DV)Ci1BV~Ss5zOu*%`04XM1rF=PBdPX{^7s`2R%`c1`P=u8)~SpQO(*#FEG>CP^{j>n1sGHM70K- zQ%xk;yWlq*mr7n~?$Ac@$(Hj zEhV!SCQzDrMtSJS*i4GlJCq5fe1DaPz(+o+%pV)Pw0@{A*ITos3*Wn_c`&o%>)Ru+ zpvqk6N9!p8B@C4$wqP42K9Q`Gx=9qySH5+38g|*@9`58PGM^NweM=1KR(RQBR;eme zznYq9J%+W;m)7hWcUWEA{^n>u%aW=`3sb#-YyT~}DK;jMyz}8rK8?hpb6?>yRJJcH z3^tEhYfs9Kb_o-?q9BmB$28Do6%8J|1-Oj2ekpCQQg zo~sAcQL6XBilOxV?S8&lNbY5Rx?kMwBMb?4EJ)j6zzZ!+3mqAl{oOWWc%K}mq71H+ zq4Hhw@U1Id3D<}A?+}o@2I{X20LwL+IR(^WYeOl?}<`;H@ELpxcw3sU|GmE2|rgL zR|R3`&nMA!y@efLbd$rt9SjGMFSDqSz%#?@0@`wneyr~mrj=>Fr(fpx%3HQWKF8V348rvXulK@O_tLWeA z0GL@4Hc!u0@;>OV{E?K^HNePiCqWGT4#2op^S&1;4`09oc)+{;PYYx4|67>EX_P{{CT=_z57JbEkl!nRz^ zfJqGdUtsg3T-)hYM=LmXrJf=R3~XjcJZS{k?Chsm8k6fhLpZwpe`tf}=?{%-Z#zu{ zIZV!%ypJvWU9<>!nAWK;wqomOa#qRY#H;piUe9lTqY$)tNfMz&Wm{3R&SD-(F4?L8 zvfttV0-Mx_5x5*ZT7jCmCrGNgH)oE}<+e_dFhO~A&UI=Z$h~XqnZLoWX$YIs4{Ja} zC`_c^i6p!WRa6DSnQhCe8jYfkbKpFW9u?K{e{y827_bjAic}e zV~+iA(7PV4SRjM?#_ny9LeXQskvLPEd~}qiB70E|5-50y{{=#yi=>?JUx7`t_5#?YWl(cM%QIu!BqCfU?phw6ylMKua7MjYhqq~{vp+nt$Ggmi83Q{#j;#m*R2npoi&DuUnTo{vn14K zcmnYKvl=eLDA+|W0tG69sLLu}b1P;FH#$K^S-R1~$XF{W^URr@M_K~nr-P*Nu{h?q z`-zi+Kv^`v__?ITq^*4C)pxNt=@N<3M6&nJDEVQla(%7XW;Yb!jvC4`2`zT4P^y<~ zqIN6@x5iPN1>B0`G?J4wYY5N;2Vh;0La*7fBY$sfQX_Ye9*JvFa5If*iw`Xraui2urzuuA`ZIf1-X?JMQs|m3ud!{ zj@M;|0Ahe=kW9z)->Of?Ocd8KW= z)EODLUU>t3X=kz#bKrP{g3Lnd8zB+~xtE)rhsJQt4o}RUMG$mFqE8zvF(u3(cr49IK++&E zO|C};oNKKWtFOb{a2^C({~9XO=Kiq&rtipbFeOC*buTjP-*<5o3gpv8LA5I zA7gpZ!KG>8sjm#*asnKNo&`}7BN~ZF$>b`qj#>0a3`bOluNy4cX6UzWm6M@3e~akV ze52Zy)E^DF-^e!UgsUKny@_Em+&}zJWRoHxmA%z_ssCIz3jQAK8FA^gQ5p#D+tvs{}_6TL^ z2O9y^GG)?lv5Dhkltyuj&cS{*l+9GRg&@4UeFwXa@xMg`mS67;41U6)qNs;Hhdl+> z8F#BPwDXeJlYZa1+}ZTOZE>?OH%H?h6i|}+b4TrA1XIo& z66&Lhfd+Y!Qi&W|vlJckBe_Lc3jdBZCpw9u*7TUfK*=*8VoA!J@pKY%vCBGU=f~lV zng#q`I(fJUN_aByV72^;c?RrZwTxAX31~db5@sqB5!>$yau;ZkQntdBWKLW?nC>+t zi}Ju}`<52lEa9R;DibE(*6ylILH2tIeygsx*w-EAye){&86eg7%T@jP{=SVw z*)%(4l<3#Fb6)~v5B2rdwY?Su3X9}-L(S+_Ob~?ZoH$gR&W{XC>nyyNqDH?nk=#&B z*`4mUu{E1ra-OaRGqW41$vpcQXz*T%=T;M^7lXcsa^THKb9qYYF+gM|;Hdp12rS6$ z4dgG5ksRhs=eTMi-e+aVw@wRQc1jPD`BxTiQ;<2hMD@N7J znB8~h`O=b_=FYd4{dFGXuf!TIY^K&8kIK@AQuD6OWLRQuUa%Y4jVb$BvOyK{P1)rV zGT^p8jiV)FGmk>ZCRm7_m8oom3H7@Bbl8(Z?f9&a&ExGe9nKw(AZlFo1MNDqjh88P zzh`$}4K5qAEiYG7M+KcT%gP_S$1#%0ZW~95UP0!Fz2;>@GV|I?T8yX1i!S3Q$;YIm2~@dp9q6|hQTO>W8RO6BEkZvr*(FdP6)Y2VD()#qoyiBbLcmk-4Z z*jKS8x?^~(I&7G6O%C=kB};duAR8r~aoiD>((^XZItq)Q+ZIZB8uYA6VaiiRCHHnM z@-ZL;V|cX4;a<&*T&SE+A{kf6)MSx&52sAWBxIFe=Qs!4slf?`NU2vSKgJZ7IH_RaIQ`k?# z?q6bWB}Zf`_OraG=to_>d|qxb4%c&LzF9!WP4b;^GhNs)h9SJfi?(oRNS#4$IgWGc z$T|Z3Nl`vZfsPm&o7dV`VzBZ7$>b&cF@(Bm#^kuQDG8v3D!l~aV`Gu7Bf^-4<-c8- z@tvI995wiyoE|DeQ*o!MZRSL2$zc)-pNEI+UVepA#S3L*oSmEYtqOR==JgbxnF_tm zBw}s-1M`8h;YvRZT*HX_K^$(9u}S=(1>{kcFQ(hO4kPZ};|h0pfsS*TxG)Ok1xbCM z6pKK?r}@49{_TLEx%dcvbMa|2h_GIY+lOqGwh0owo(T_3=4>+)0#6!yu4-F>BR%(1 z>ICJYyh==(Dt13mPJZc^DDRRQ?E`ObsEWxSPck-#xFRB?gjU$wI-8OzKZqC@3{r#( zGCGQ1CO29wFlJ$BH{raTh^Kj-a2V^jV(z?s8ekSff)9CSTXp0)j{iATyIRB6fQ+6aHcFvlUf)B4jR* z313HBqhj;gn*ZmyPO5^9We}Reqe3V%!EbPT6FU03tmzp6LQNT8iN_Y&J`MY*raqlp zZG-Q6i!q)QP3HaFeSaWizk*@WSegVLzwpfH0Q3{1z2P(sMvox(=C6x3K@G*}Fr2nC zkz5!SOlBD<>ehsa?b6QWW|=nm;yXKbO}3rU<;0bU-H#am#2MzFbD@UYlp=;b54urD z?rHWMrY`)&&Tr}Wv{fI} zKC^_SXwD)gZF;~9gLNlM496TYk7}y>aWv?QI7D-n`WdlqjUOFXv@Fhr>0Yw6x9tx( z_EgCE-0(}b5tG|l-S3y!%t!N`ZeJqy*YQd8H?fEQc!-FiMJdkmkbUEc9ws`0Kj-{< zP=zVgss1Hj>RjouI%J@J=$fI)+eg5tVO3zZ*{n&{&Yae<8|ah+bY|?8KeFc|un)l^ zUQ|?;|F|<2CfDJeK;DP&92n>c_l;-^{d`_VuC&TdFm#eEN~cp?q3%%H;K zb7iI~LU3;CHYYw_CA9Bnes=PUuF{GBHu!a{SBsO760L{GOp<^0-C<0FMpz9TcM6l- z`sif*JAtvuKWQdz7K%LHNG`!k*a0KD$kbHKD*&H8r*cXaSx-jq9K-a4LuXIwyofSS z!63J{`{VUtd2*6-wARqq3L{w7Pf_6W=xE0r1MLGYI^^8EO;MEcc&BrN{^%`t@a!&Kl0cRpkbjM_0O7-xjd6H z(IB!>n}=|!P!+RNxn3zf3ZEZ3#m{QV5yZd<3DKV<$*=A!EEm$w(64W`QE#HB)$}r! z%_TQjg+3FxU+)-tlKxmJ z=E@(xE7c5D*Qudb0soe?zqg@t{1-I?dAG9}?SFp=RM|YgdV_+RKc1Opi-6+FXYJm6 zD^F@Sm|{0KA8s?(c?6SUW(ngJ`|9=;T=wxF+}gJ(VvE>Fo6zc2`!oG|csMf`SJ=s@qHGo{BfGO zDunanyy`#Jj1a3x|6r@?2`f$hA=st40@p}wpPLSeM9Wn&QNlc|Ai ziywwV>OCHW9~CE3JAnc|LWU90{7bV78px(#oGNb{&Eh8q21egJ9!?bQ24-eLZZ`(m(vz$U*L1+1P;AtKAg`~v%v~4FK1#a1e?5wIB_1% ztSpPWdyM#|4Z*iN;7b>=Mh2xP0_Um%%dW+cMeObf$x}o?LN3@g)lP%x4;f%i+{hAL zYGO;IY?6=Lk649Fx66GwbH0^!8uBQ4Rvcth zr|U}%?33tTLa(nhEx=d$P?BZxTr}P4XxF1(on{h z6?TCG;Qr$K{hh4>2UKDnD2D-xd>gWgRXN!DKu_FvmaWuimJ$9Dd7k5HHv{ zi!SumL)d*Ob%OaVqD%73PRcDRt}`u*_*(92ZMG*9m^3G+GCj$b$hZixCll#V-~Arz zOf#e$y1fd`fu<$>;2|Mp!S_nk5%8Lf}RAbsVP+>-S2ssv8-E)?OyM09{ zGeEd4zBoC~fihv&Xv(5wigU)L!8QkU<5%{tkK#xN5~Mt7Au)1#&VCKlBQ0{U6W1`d z*t2S>1O?O?d+nXuxgE;@RO1*;FrptR4OwV^fFHFJ@lT;cGNA@XokKdf2rztxV_Q_e z!?fqa8YzNc&5o<^q;)I{Vri-Ja9bM=KB3Gx4xzzsS9ss=m`m!SWr#dGC4E4xZ>HQ2tCtpsti(f`lMhDG z*rt{WrArW%lZL`U$dLXOSSv0eJ0P^-QxYg{IE~F<$lNp*1FyM%OedO^py-{kg0IgW z)T?L5J);}s@G6q*O-bq`A4Wbon0bcy+K^IwzpZ7-0_iPqHFX%tYGX@GE&CGNSRb&R zf_+;N0m#bj_4=C^>zr^vcam3BeCq+-Em%aXB8lTdQHz-@q)s{+%x0A1PXQ|JyrTSN zkdqvm`?fb8w3^Drh6p3ABth{I8XPFbC4;~P{L=Z1a+|~4o2a3j@LGE55+p`yi*9U( zk`?7~xAtafZfXjeAqN>j{3+Hsi3%CJ%7@lzd+4dE6Uf9`o%Pm@* z*i$~vJ-WM9<0KuJ3>SBc!XAj8Cjy)J?j_E01U+w@c?fh%P8ySy}@mK8RsQscwt{lfXuA zze6(j!(w7R5VdmQi?>T1;G?!yoh_PpKvg3b)AA?PxrS0QE5~*J<&MTd#2y%Hh(IVTy6ez8 zBpi?pX^#zYIFfilWukZfmog(JBbH9Ao_-S*pBR0ZZ8m{l&9FD&=pYpQQAwd{iVF=A z4RMM4k=CHWl{cSp4&o!h2m{{$!;oDxdwL|vCv6faz?+vaN=OE{QTG!Lpam*31+y?M zb~*o)8E`OAq+l0Wkq7O6dCe^c0%@zYc=5lyMrrJF;5iw%4QwXKzMg4BmRKyDMU#d6 zF^(m5?fxrHLUC4BI#jf&^Vh>*3YH4#2{5nm1CfsgmL`yI*!u0)DPoVcEEjWfYQ6EQ zEm)OTJvu-O(eT5&x@MPe0ei@0F2iVsfrrsjsc)DB$q~9NK@gU+QX!<}esb#IUtV(t z%xgRh`s-3BgHHR)!PNk@xeB zt1Is3P-yhS(e8gz?%|Uv-yVs&uKj70sz$ofFQmw0wU58G)bVeOV_qsl=w1phaEMyO zye!(hkVRx=qml_BCOrmBY>Fa7eXel3b|Fegxy3&XF@#g74Bqu{8q70gPk8Q3Jh#En8(BGQHS-4hlyg>+G=EPx#T4V|2Z?x z^D-{*a=qdA;Xf%#bb!vxt!vtxQUupsX8{gKKn80(7o1T`h|D`{$J>vazhQ4&rgUNg z7*}Yz365&8^p|r5+y4R^>=f}E6T_Y{8y5%MzA0A*jvz)uZ#T4Zms$Y`CA)3K_?rD| z_3@}0nR0Rh$T-KXKc`da)!xCF!K_kj(YN7>)kG-Fx@N@=3boiMI*{X@8EP!yt{&vi z;>et}O*KKk@{yF$sc3}ZJSx(FvoAQp(Ok70k9dhccf_(!Dv(v=>h+D1oxazt&R+KB zR%CuPl;k|TpiPS+C9yMtBo=Z0zS{WM3EKguzlZhj7on7)Y9FE4`MEkF>@uW-F zT75(!)&LeZC6CHHq^OnGB?RHr3R=rlQxdBzuf~|dhW(k~%nbsO{-R`JZ8UmGN?P65 zU^(RDmnnRT()_r(^IhW6Nq^Hr6C*6hkY#EsNPdtXRCMFdfTHmo6^0ZT_E~BjLgL*G zJDz+wn+}3Wf$VEVs2h=|gEX_3u<4=ylV(Jjj(HTjW%qZ~|0B&*!G;rl<9{X)7Lb>Q z?ZN+-*mzUpudtmbD0PhC0Hqlfk!$g6TgV^u!V#>hM+^`*>=`8J{}P*}e8selg%_i1 zwsW(zoe#AhbrDu}Gbp&eK`PXVq~#yNos_U1KPa}eZE@kNkdH=P9B*ZIt|j`@Ay%Eo z^g;{JMrRiS$;jVSHenqGu)*BBWqQE8j7={yp-7?aFgq&6vW{aU^iGSd#M;yd6-!3w zh@$swX^KvSk!5|66z62I?-bbK%LeEa#_ZQ`b(5+gqVg!yqP(?`AZ{b9#^Aftu5q_9$RZw$>eJdi?ksmnJ=5t9r z@#kJwGRr_~hIT{}b6*Q5@kg>lMmL>pk~K@9=nvX3@qAlc78zh<6JzqukK8}m$_&=i zVD_^I-K5zR8c(}71{UjIWYa_he*vPimP`-*FS4nO1zI!mia=`y$A70M^M6`1>`D#T zwm1K*nY0dDbGJ3g9?TKdJ=n4?Q_|{|3dp+?=SHzRlx!5oj6K#4Pe4Ivm-#l6YLuz} zx2}yjQM=e+#6BWwgAOT?#NqrBW^_plwkqmt$bXT|E<_x}21n_R4F$EUTAdzMhI&}uV-Tzwq4b4?!)Emlo2l$r zRLRq<|5w@EakUs-3@8Zyl+`3&%U|jMRyJqFJ;3*0)&5Sc?cYKElr>-{m)Vg_hC)nM zS7h7JGccuO&DVV=ZIwC1;wpKJb+Yp%D$p{ZK+)43AXl?v!!#X$T=u7+n`K(l*>%Gf z%`%=0({9!>FbHBcLnSA^V6rrYkEge*Q{{_zq<1_n%Cn*f7tWQmjlWXMG1%7{`>LZw zDQx?tWQt8$RvLDs8fq_=6ljMEwGFlqIdx77)Mk1d^^z($DCHa`p1HSa-C-k--DqT7 zv4EKk!X#dcLwrKeH|aupLvb9SHZw7W#NO|oFAs(D7EP4cXD+>q;j85@)Rz_O2nT1# zO@|WYU?Vxecy1G=3mp1T@}Hw4%V3Nca%|a=up-1@!CY(8qJ1$}b@)t((#Qi|**7AG z<-vP!fjpBtLcViE4uV(^PZ1$GCDz*!a?d~#myFt+O?89itv1xtGougq|8LsNPT>9f zTgD#XXS%y%_+qwc4%*+149x5L+o?`KcdubDpmF1(`V&BS7@$PT@+T)q$=?}oF#(9p zSXv-lHpl6~P#DvLU2|~K)Qv8l%qq?9@IQ1KV19;sei_68UYB%I8UzB{JV!L86F*o_ z!i-ukX`#uzxJf_4EI0Bk6vcuXHIr$q6^E|p6qFDbYQq@%t#rOF-@0pdTn83jOXFfg zl5%&WvhD5~a|%Q?uVSfh)&`1YlY&MI6YRKoR!u!+8X; z#_uNKB)cy18BO?j%>G-nMJ>p@x!V@!i8hK7kSP{iGWhw)KUyK)^|{Ywg`pf?ifb|= z)~6908iRPHM&|8!F*9h17H#lxx}U-*8-qX<0M3Kmr>#=8LtnOMS?R^fRygwYEo_fT zEjH$D?FIsJ^6YRe{=1k#0b=re`p@>i`escv&qN|((^h(t!Dv^hn`0e_z+4 z(Q8vQ`2r9IP=BpM9-8?}(omFMEJJmDV%OQZR^c%*8A}SvE^G|F+{FcsW64d~%C%maD676Lw@E1XLf)l=IVIztq!Nsd zs=2e0c{}Zry0a7^D{{)V^#>Og!Cde{D!dAK-2W&$Ks#^~^T-nV*7GUso1eQb43*|=UR)@1D z=RoV+?zr%DOMib00Z5{#yB=e|5DL!fQe{eviKHUK|KXQesnwcmyKs>6df<)f3|4z9 zVF8{4VkpYEt(UCH*jFulyrYTqNB#-Q+JjOqy=|u;gXFJs;?~&O8ZL53e2MlC6q%RjyD zXf`I3(@JBVVj*hY5bjZnPX!Fo|Au$Ab*$SBumg8 z^5?>B#L|}tB_ovA>^L(>%-y^Ic3r0qy@kU5H%0m{vB+D zoUWJhI<2gs5c^KAmt}!0pV04N)-2*=))jWxR?VYgE=0b^d-B=i7fbi|T^ZyFFqPv3 zrbgq2*Mqn($moODdqu&o(;%aJoXnZE`xC>KIipB-;8655+;f08Q?=D4-+HAT{0H& zM)O=E1o@q}@ZleBo+$`hqEjb(U#W{B+lKQG5J8O8<0;t4{5+2(e#mX&KL%~@Q>_YW zD>!tKwV`J!=QYRtHUy0m)$mbWxfxuwcC03|tu2aFlPPAitxa60lSbshYfWsReP5a= zhu(>1;I?PrK4Bk|dT<_Zt){}(XGt_fD&_uR7!jKaj&>?7@{%dH?PQ&}OAmr9vb}Fd zxU)+&5|mE2{nZOgzAYgw_4D2GXW=ZGPZZwo;sD^2O)@9R^ssl4)sjZ3_f6-!s}qWy z!D-+Ws{x_U6Iqx5EtKQ;{3MgTIZCirv7%s@)~tg@Rdh_e0vi@~8}oTYK?^-JJGYwT zRE{8!r*u!hCm9EY85C*z!f1-!UFV=iDLp|jXjl&vBgyL-v}_hJR)yE|9!&uXv|wzx zY^7u#vi1brv(VItogRTsZ%0ptGT|eFSxT#trS;q4;h(*^y-ZP`+4IueuVDGT@fA}M z?LXo*2<1!nJ8wbb(~U>$tuQjb)92(q-a3KfQ>rq+nquwQs==5+eh6p?#{BRwAYf%g zPukZ{su-nwz%JgKF&P2e*N!)-;Gk-7MnP&DTp3VYqU8Y8&2N62s1cY8F;k41m0n^T zBs(zx48th!4TSvAOh3VDYUXX@WT~;hT8bvn`(}-4<*#DxDZrb+SJ@`)3 zb3Sc4)1ZQb^G8rfsv@x|rG}QoLY?~2PT|#}2$Xf$Ts{XC(T?&#C1t=V`)J^5(53MJ z71@#|zrA-JY)b?Yof$Tp0s&ePrg{KofO)KuJK>8`6`e_Lcorm$KRX=V+uJQNSONr{ zWEW+q$p2I%AkwFBaVRQLfae&IY9ZlE=!l$kz5#UMC$gpZFntlM=W7}~p*}K+ad-%9 z(yIpUA3qE5PYw$8q2$5s0D>24^l$rC*d_sEvJ2emy3@1DBX>K6-XBqxW9w92SFIRM z9CBgusm?##*g}R+$_;(Gf9I&e#C{@mh2aIW+_%c;-`(m5rbx|8c0%t|JY$M;B>vjbhoV~I$`Xr6wSGZTgOp(hWd%2|-8DGoJb_Zz z*#}t(TXn|k*)$ANwO?svC+$Fe?+3}PF`)St6#u*ubX8#P|R-2x}je3N7pwwefO6hAk74|jLgJ0cjgyl zM&sAFZ#Z9m@6erFk8ghA7NW!4;9$v|bAQmvb(}yn4yLrb`CZsk_AYr>9B-EVWkij? zribN}ESOw~Ns@SH$`o_#B)ipr0Q%H%!wQ}lXZx}9qLz1qs2K1sxw^*uhaNO~IBvd; zQQ0#O6N?52K#mcu-)#zGGwZsv@Z!m#_1m=vzq<1|YW$LClwPI2vQ`t>eX1ie;@OCjRPCCN7tn{O0GzY;l_x)p>#>yk0$zGrP*z40@DJpG@BGGz@ORp zsH~`Vd4;G`Gsw#bv2C0w!`c!{>iOFcZahH#YJt|1AH;k{+Q)-DFd0{YV5(VfH^~&- zpcj#nYNuoyV4#gWG>@$p6Bcsps#5-kxhinCx6=+*&U1 z0>vlp%^;3Um|e<*yz?3pzEa>l^S&Fjzv1jpi~*LT23FDcckep5?G?mLUAzTFH0#cD zK8h2wi%!p1O1kcdw`?%d5V(v_v8@eG)eqYrp(d_hUl^$?*iBh34J<5Vr#ud)zf#U7 zAjDi~p7^jPaQ-2WUSJ$58;7|NeJA=2;6729JfDb*p#D zeGpEx%+z*$4~Y?B&=d1Xk-Zvo!@#OjMJBcx;*NAiVoPdSv-Uu0ntFWrow|N1*LhoA ztcHb4PMEd{NoIAq7?`?B0J{lpg5Khz&(8H?%aFVLP7|lyyIONPWA_1HM&WX%f$P_+ z*YE1rp`n03G}id-cX1NNCN`wiEUS6avC(qck;a(fhB!zccO3GNuGt3+*$`7(B3bOO zYMXrL)2T{FfK~(qX>(hS?af~skps6rPq}YvVO`j>__F}eai{)v-Nvu9+3b5jzwggl zw=co9jUC-Q0O5Yc4OPjAVHozU>C7Xdiwaaqbl8H9--MQM1HRAA`dD;O$n5)gy;af) zYGEU2i?44-8+j0O$!Z|wWH$N4fL6N;G>3_}Q$AmVe^{X!UH{-2M#5W>K)T= z2)B#&Va)a3%7~HbtQM>KV3p>m^sYo#Mo!ulR6A0&d5006o8I6bNQdS%&bpP(vDxr_ zh+6WH(w6P0_Z|Yw$J*kfd#6=$zmZUgPkigGIDjiK^ql`d1U-31#!w@%B=%}U%81&1 zu@({FUY1z%eLq04QRrx@^2CRZzg9HpgsWp5pZ&>k0!BqSx{y^c@qs$-7(A;|VBB#4 z+D$wnABM9!S%H-e^ZNKV8?n@UdcpdEz-WGD;?y91Vpciuz`dhrTzG4w!rmbhWuGlE zkxMl!VRSFT#;r}ntqmFN!UvKbr2HW3h2!uDtz8#RzJW*OjSp9#6&@4dWUx$O1o+`4}IcqUSeQ5Qv!7gaa2 z7j0x2_>EXG1qVnXik#?qsJ?-ptTR!?flP9<^WgG6lj4Y5l-k~~AFdBl0w2a9@WAX? z0i`9nMHG`oasAzDorV)De|DC})n3$OH17nMc{KBDvPk(b!#2s+R-XD_07p-4uE*Doq+~Y^;D!M2y5<%5X45bR zP>bCgavNodm23X4LI_@fNb?-SPwLF{5uDB>C1wI5PW(ajvxo;~E8dofHg5O44EpQo z?3mw(IBbSWMXe!P;9z#TkW(mIQZqcXxM(;O_435D4z>g%dQmy9IZ5hu|7C zxKmT!ok6N8gme{z zv^iqhFPb6Kq#R3eY`+x)eXHHWmyy%@+g@ zHO4cDW7Y_=wg2urZ#LXcPyO*>p6+<3K5^Mw*Z16aXeg+5*USauau2MW{T-s-*e5s@ zleZkh9iQb(euj3Vw#UUFJsapx6hN8*94v4+F7-jF_G`(lBNF&Vvk~!IO^#PF@JTMZ z(u3=KU@}Lb+tQ@Vol1FKU}f3LOXx(;oiEz`*W$8=S7ej4T?+8p=d&uz?@8xFWm37@`x;Yf#8D0+z_k&6nx?K1{`0JPT7-rl@?T}t z6GR_8!mAX@@>HaHDQ9-{bQS@6)L(R5N@lY5wCLGIissY!b5lAa`V0LtGCw{Zm&3<$ zYA6bLz6hqs5*U4z<)ouV{r)RV$senZgf41SC~x@!sO1&p8WYaRI{O*u(yrYGB)O`6 z0>XIn4tur|$=DMj%@>>SL0&`;UlPNT@|ZW8h7e7mPLEP{#%Cfr(%6cILCDxrI_aJb zJ-%aTc(n>VAOvx%FCDAhj<0NdGu6bdc8wux_$qzytSW>f`FDUF3<&x+*X-F#;2k9I z&S#*G2=J1m`Z5sZ_PrMOI{QEm*YL-p=#MprVtwyxCy6V{Jv`DtE!}yd)!t;JfF6jx zi-(=f%gdEJAkjtbB@pFKgNvSyVbu~(aJ7|DF<#a@xPoORX_AQ_W1Cs#hdj~E3E@!< z(rl33_y|d^bM;!dcm>G>rq@Ud*MpizF$HZ=shi%Yp3|;!c#P_ov^$gc3DHfnDEpD^ zippN3&`63)jBKA53~isTI}86hnE^Qq{z~ueR51f@Y(+#uE_}|YXQv<0)f+nY1RYd4l{_XWP-0>!V^nlfq*}-RNBNH*%dPV#OAYoTjffJ*fyx4l{XU%kS zM4=W&8Q$BhxPcmRte?d=^7GnO{CUw z(8W;l9D4zN-(SnO@Dku5%uD35Gu8MUd5g>bDdC*V$E(kN3;0L!Ec&$^S$pO^9|4}Uj*N;Z zf_Cr(Mz;!D)vdo|)#}lu7jkkL+|srzU5Ce1x3HQ~MonnR?B2}L=ZrV}Y=z%CwAk6Q z9|-91PZ;-&ywHxvfiV`datz;i`f7wiWyhWJ}7kKW>pP8hi4(_|ImV~&Ha zf6cA}K<6N)m&ohbO9Bpr*{LIoO5325 zmAix$1BXs=xB#(MrPk}yg@dy7N{s-U_5N@m=C}3uL7HVm+FB;HP=AlF^<(Y@5|E; z&wB#_pY5&rCaUYz-m)rd0`?wT>)D-L(0DI0Hm4D6^r3HYbvS5j zE-DaF>s3*>iXS+A3zjnSFQR;5G8{m z72j2+SwYevpe?VC0cnmeu#6b!?RD*v3w$!oE5$pO8fqjzl0PjYCrtjm#PP#EAeh5S zXgR}}-gYLNsAf>6hTFFvTwNdjz4B zNi;nn%D(5TW47qk1aMtE892ld3B0}}dc=PRnh*hB#E2z6Ino@Kb1wW^&*|Mhm0+Xw zV^16)myJ%Uo97W=6mXiq<(+79XWG@c?Ub(=F4x|`9KF_D4RLFmeO^_2EZsdjv95AC zhltEDs^eWI!4W|(9g#beC82`H5sXvK+~LN z&&THv;i?UwqZ5T3dS)4L$No9+;)Z=$QvosX+~yK^BdTl&bnkr+wEt7vI2Ip1fA!|`Rkx1(5Hy98eP*N+VjM4U z2OO$`EwADkuO?o0PylhPW}pm3k8glZZPdWatLvJBK#{4%+DH58H(GJ1_s!cUtT%$= zMGYrdU2!+?H(O$kTtu~qO}phz0ApfA&bYi(Or6y2$Ei!p>w9 zCh%#LG7mJHv$Dt%K3)eR`#ikmclvhP17~5px##d^+yPXGN0`waQf=md0x>jD28e06 zpNkmnSbzZbcBp|I$`-YOE!vzj=8vdBWh`JGc|s=c?lgilD%P-PRQA-NAS^D(vS6B% zV(_$Rg6*k{iR2@DNL>=6d|XVNZd`?sYRKyy2PXrEZ#t?eSn1S49OHg91p!kU&;4CB z)oa)IOV0tyXV$k};|DXrv%i5$%AP466A{q7?aXSNab^QKj`>@qGp!;_7G%@nidlpN(BKxFt* zWv3y5U#|pOkAQxd_xCW})?K8RY!S2<*Y^q0J|){YI}_-T@$gNM)shz=6>XEre16D8 zsp<`}!OgQ4VT552ohzWwPM<1bg3EVOLvQ%^YnrYX1}PoZa76nZnNU>wXbs&N>eg%s zQWSIazgujW@(~Xet2>&?d!{D3){2^KEP|=dgr*JF9yqwXlH$F5MQfm?nR1P*IDP5< z0EV##QuDor4`F!JHhM^g6)sxH5S^Tz(HFoKvAOaV&{EJz@ggRRk$lNjfRD1MiLib< zon-rp8F3L!Mo@!o8|6OW>ya~wAU^lT@NrWqNfda})IbU^N(?*!WSqgG9~tgbN|GUw z?giKP84c(hZJ-{6d(cd?NjFZt-DT7n5gT`P`MJNp9>JVPF-;B;2DS6(M0KK&fL5Qq zZF8VmR5}E;+>hK@qB_R5zN8(hdGJtX^tbw9m!J_2_ClU8GNUNKKOb>lYC4Yzx47{DZdD%o6 zyR(Qq#{7YG63RF+4f7qMyToQ^=bgCfNlYjI|^=M z6pu;T+yml7*h5p%!WC-TP?O!>z|~1w1%%n5W?D?M^^nZXQXgKy;P}UnF zUu0F|D|>kydW~U=3m-=zLvfWKlAU6649Uz;iJ>3fq+VlpNoE}?Rv}U5r*Vc-KWDRI z9zCMnaihD%(w`%3?J|a#X}4tFYoB@&4l{mT4J*!(af5)n!4yp`XB|xlU~cqNXoQ;{ zONi!Nj_>*F7fue{G^8JcjsWJzT%f?*s+gmos*r*XIVjll=g`f;48+hc-(ts&Vvx!ISc#% zf_cwe3fi6oXQ>Kpq$e14%aDos1h1x+R^ADcEjFq!wj@W-)-k5#P)cPry<-&7)liQ4 zXs{GrSn_GU9kXe7QT|v`e&QiEDpgVX13s0vv;Y+73x%qBE55*Cb`lh!G~5Q;P*)(6 zlhzj=g`-q5^NGi`Td!ZJwyp zDq4M<9BLHq<61hsL?~K$zfPo-`zD8 z|4{NNGpR0O)w6S@PF#s`%n&Pp6|W$o!38BMMIJvzk%913(o3qGxB&0B#|v;h0`-x3 zU$?lST-sbNL(N?c<2^M|xKq$`JZa>d3`9|SL~8TAcYk8Zj4Q6RpOeefi&+N+w1+sQ zqKS^HM>z&|kTc!SaH}5>uV_f{qyW0?UAn01J~lcjj2YI{YB>%QN{pP_0gUjz@9>+* zGCgo)WMj3I5cJ!7U6{{R1>-8DN)c>Y|^CjR0(U@!7DdaOnoZQTQG=dH*UFt9BpAZ#fx$=ja#0 zW`NC~n^8@sQ)SiMEPO`{#L!seIYm{-&Ryoy6V=QN#xJKOze!=#yA{MXJvGN`kvjeJ zuqwE&;$5ETg35F(9ONYBg+t)!`s{S z-rqno4{>IwiwqOR@B0DX^gr{KQlkG5R-KNLe7L_nyxiT2S0%IqV9WT7zkIn(p#Eg!dfNhIhy|Ve3;O*)|NT zLN+58bXJ7;EeSh@zgdTZ=G{FDl-lI9#&*nJUw2XB{sxgDre}bSm^em7@kLSG^M6`6 zLUr%_@LOBBbi(7?supDe$ErgF(-g~^OM3s4MG(&B>a~R86I)~eJhU$T4#&zAjrRv> zypgv0*zOQDs~6gz^E15g5`g>U z$wZ3U7TDibi)@B@!hsP!pI3!D55G>oF5Nx(yw8@cRHi)Xk1nCnACvv; zS**W%MYkgPXgyt2W0kP4VF+80 zsjv8zdh*y55eB z>1G-e9_)JFYUf2{30bH=Ax{9f{5WENOK&xd;EeRE&K(NOYjnFwQwRXAQxgg^QAd8c z9L;zjqA+T(5K5v>%pglrc8nhpe?3H(RY#Og^K~YoDL9}MkDM2`$R0eMH#G30Qtm3s zyi3J6R{b5sPVPF0!NwGa*Nusa*yu%tC*7r1z#7Ua?>PiD$h+xjk<>kQ!FC*aDwZOu zk_jlf(C>b7?fD~^qT_ERN06J__Dux!XTjo1QzB#Gi<3W~Ly-HUAreT67w<`Wc=Joj zl~s;>IYs~kp+{O!O8zLUkGVNE&j~SAMOF^u+ule%Kz-2M>3a>=0#`8kIm*L%vH%28 z5#O80Wru?LYm2at5?GyMORz=_)-00(v!2WqXsn@lgaX^nz4Y}oX29}rtkhW;0J$-A zBM!0gwrG(B`-JM!fQgYZxU$6Uz@VOqic5+NdE8E+)pb!BB?Xet6+1fDz2em^p|v!U z!^g@AGTHZ6YjdGAk||Vo`W}OBg?^c3mQP9wGTB?$QH44criSvp{zhnFc{yX)y%G#I zPO$ydADm@zV>H6#-l$G|zdT4fUY)nFe~+`R(a*)IncS>kkRx(Igy;+#V_vyz%@#ez7Zq@e~n_3t9! z+yD;`Omk=x&&TxD*uw??CB;PoCEe>eW##EQ3hB}i8-hpC zHG*cU*P=gDA&U4%zv{h;jWBfeT&{BT->VrJhq?H~koncnueSpr(3T2lNk zs}j0;6E5Gp;QKO#qD(UQFVucpfc{h2^mzqbN{mzx_>GrTyessse8(Sx8IlIgfu2AN z0>7iA$j~-n|Lzc-#!1PF9{$PlQ;_XjX9+@~O6%8VClEsgd~fy$Am>j5@Lku18rhg` zchyG-`@0fO%R1awY42R}!jCKmq+$*>0MybmjOC6RyF6CE{tt0be=tCwc1g=Om2ez% zwZ7E!?m#0FA4>1qi(jt%PS}6p_x>af3jABXu7N# z7&(}(7o^=Fvk>I7suGQf0LPJ3LKWGxlPk4>Is0v5h?fd`C0l;vB8O3ToA065N9_*A z^u*aN0bLC^y7+p8WY72+Z`BtE-g=6z{nG=X)LTPE$uQeNF;QGz1X}*~=z13+mwVx1 zo-CyMkO@QG1eMppDDho+2*<3T53e8fyq#P^lC0%XScATzoESpU-2EreeO+L}T8@6( zT{1g_)vM)Q+O)Zl;-HTbsNBIhL3+{b$vR+zdystbwb6II+0?GC?Z=`$Ak}QoWTJw; zY!@R!V~q%%p1C4vm+@COf(zrKe)Kqm5dI>g)yx{VO3i>hAW=U444R*Y*TRiebVRP; z_*>p9(+vc~EH-McJ#Z+S$UMM`$aGn1A(!JX&&yV+J3f$lHPsU`EEkaX>E` z4*0108>QGDc?Bt-NpYt#?`r=cTU+YX=$@w}wH-oPrku>G<8~EOgZ+WrngjJC2j@3k zgUSdj;@G#sk&5+7i%~yezprR)4h#cgn2+kF!X!Vd=ocvCjcglO;LfEi0#zg*=w2)K zC8F$J1+yN%jt}VPBKeJ9)Du>&Xbl!-qJXNzI91U5RpR$uvq$WmAf{J^tqK+^OM| z#wDsKnFC}o;9wxh1PP_#!Xj`CL$1`UBGqtAO8@sJCQw>>-nUI_9ZC|CGuKVHPs#d8 zF#0((((fp)5;h-QyIe#@_TK_s#fKO4R;~mW~ zMr%&>x!h1MTh)lR&GOx3y)I&|I)`;cj6*=Eh|F60mNgUS*6G(8JS)7JUf2;+wP|Xy zKC%wz5!B{AE@E5%>|@{wr2->8WTRlLpOis9XI=&I5XVQ}Nv5?(NpP2rP`TsX2%@@l zJUJWtQ=IY$j0}^&TG6#M+?^QN*#+Gtg5Ds12tJg;d!(|}RBAa>6JI^ygU~!;eO$YY z8qsmwGgD9(1JNy3t&6~%fgoT)MNI*k5`vw~#MNve()i_^V@+yzsFf^?q6Y{!$Zaha z-lJQpBkyLgK zP~aSmD1BJ}c)NNsgeeTq%wV)KHLm||a>CRuIgS$5z)$_7bp(oU4V_UwzBJstVwO7s zYUek~&VJ2lV7|332<8DlbyL_+wV0Y+6xe2qdQn{0wgv0LM@WK(r9s(?2=l{>j@ zeIn4DWOOV_;y<)D>4YV{h>al`Ly%)(MLV9ofbxGa%hryk4JwIVfz_^^!L}0Iks?)# zKy2gI{gI`X(eA8y!Qp?ICRxxk_tM^CSPD2o4b$8TPNKV80Ef#M8}4recqvwKX!Kk0 z=6;9UWX}mZ=&md8G91DwRB6aQ|_(H*Lfb)hrqg27jFnaq(%a z{a-y!dNzIOx=E(*7f=j{{dF*v?z{W9qkpEBJNqnR@8&{H%!v@W^75104JvHB3k;8h68I+)>YMk>b>AEZMSa?P= zKf)D)iGrU>9Ibf1f_iD_S0!g^fQZxYRc>|-je_&w8;~K+_EuF?8jV;Nf>r|^HrR(& zWNjT@zgDKbwoN9am%n2>taA&yFKhEy{n$w{zC+9Im|dFq^>jA2pVPcB4~tSDHA*4Z zdFeLZEmlRHR~4*T@;|wqDnd#c`54!7|FzS!>kqXAe}OJckif<>lU)OrPiWbjMtHITZ8g zK=HNgj-2_oGP?L8lh=ER<*m?Ol(Z@a%xKt9aq?&m-o>6kJyx{}xKb3?9c#E6YeQ_- zCjU(?K`)hzzA=n!T9youx8|AG5bJb+3;JoBDEartoRuKzq3XjI4WDK`YCs2GkhKlVJ5AQrj2L09FmnWG%+LMWC8I1)-y>2q&=P`T8pmEA~A0;{iOEhNVqpqDO)bXEe zC_-2~Ia=KLZp?!Bm>$_IKQAhLzEDQpvHjl_O|d5brD*a$0Zg!^J|%15M#aWz?~?=C z1t?hPK5ri{vxYea;M)fB3}o&Sa`LUHOAWSO>V&B0x}HV0jN0;mH>ZD1{Qt>2TxSA# z74b4^^zHDzyv9QhQQQ6=;o&m6E&8=?!W@p}^b@-Zcm{GFCP&>B|B}RfF|thcKJ0n){%kJVOZT=DRbwhbA(ZJE zb4I0F$JVmX_EHOD?#f(*%EA9CXgY(e+ToFeZ9g^L#CEi{g*&fu$c~V4{Cu+ ze}}`Mwy^O(`{0|<`P5HM3`Enk=m(v(sg&!GCIDvSzw7xJ_H}alQ!S-6D*QPQsizCm zZEJ(E{R7+P8!y>^MwD~kujqpbyS$Hdi4g>N=Yvq&1!t!8164R@?@*auT7RXp2hA0V zqE?OnY@nf5V2`ekwNG7<%@TZ4DC7yPzaOo}wq=xA>!&&ImhWTfzF&+VufNm*-<&h9 zXN(;+H`-J>W4Mh>2Q7>Esl;%ty0(mGcbuldvsS-EFCFL)G?kuKyz+!5X|SuOuKK}9 z!nLK#_wq(o+5i74YTEzbh?=z^e-SR7dv<;ktT|Oc8Dwvy2 zW+57UbI>FPM~SO`XoQ+0c56qmfpAg&2lJkf9=x-CiD_)cv3VeQ>8rAEZkJ%7@5|jE zhGdLe`6g0>SKr$|EG$6orp(=pxqa6n#UhWF%l%ca6B?F$p>;xOSz1{FlPgk`E}IBg za1)9)II7n;+!J@0A~5OhZ<9=4hB~B)O>Q27l(014neLD3y{hhEdv?R}erh`_(dZf%?x>5fQq(_U z8Q2Xbc}u?gNCYf9#QDINBmESs&?U7BRS5sj--M>Dd2^_j*VnU*JMh26^TYn%dzqyF zpu-kq@#bXSpr$QiO3LKc@c^@UERrMX5p7&S<+EQj4(zoVNX`6{dW64H4m8b_zv4H6 z>lVdOC&}lM+oia*aWj35qn5s2YrcGMsHbJT(8K=X2!$;LH8F7!(`{Jw(^0o$14I9o z@#dFNfz?{}>R=V=5ZM$o8w#LnT!?WH*x9~~VTs7*PZ?0i~dy5nk}&_^-p#emTv6ply0j^QwVw8-q)>y>r(YM`R8keJx(AQD=$QFtZB)?3lk# zcS5N7?FX1??VWzp8lh;$UxOyO!kx+ky=DABlBy=rNy0j=rny^T`-+77Zz73Y42n$G z#wvAfL0g>l=pvK@F zS6wjYz>@B8Kj85_$SN!((z$HLT0?XkYHd2lCC)hbvRN|D;Yh^tO6mU3;`?J9X(Be? z{j~=eIS3$21!rXT(}yNcnVMnTz;;KEVt*JL2Ap&!E@s(IgtQX>+Y=aYMzO*;Rn z*Tt?RSccf4>K1x176n7^+8Lf+vsZ;|S&oBHa!$@sE4H*x_eaP|eHfHJW0>M^^Ry?K z?Fl{Pp6)VljI=gj^qRL)0@4)8zS*=QUSyfR-RtDtaMQaDsJ<&m;qVwgz86c8WeezP*=$9@P%1xhCeV6wsUT)O`$#$8s%+J(BI$mJNFR z+pkuz&qCtbzm$#ABz?cPmwKkQYc4ZuUJn>Y9nTOrg?qteLu=G+)eW{ud6&Qm zF4QEY^FqjFzB_ymmq~^lCrC+7oFS#=2gIV}rpej{YRcLcE(9UkmQHFmRll%)b02+_ zDg3iC3n9p`3#5}@CU&9u+!4CqegV#JeBP1HuI>US*B*GLAaoc+o&x(3J2X23%S>-@ z_N)ts$r8*Cdx$XTUd}L0$fywc2@VL733}!c`Gp0R_5PL5V$PakIkIkaPX3k8ra{~0 zv5qEv%N&l+fQuJn&yP2AbmgC_;v9`GBzG&oej}mJlPgu{!0u0yeGu}%J1a}gz8M{j zB^Uk=@<`+QFUW)4odrYo5!cKdeKe}hxo|c5e-Iu)Da|*;G=)vU=P&@=2#7_i_#=2+JCc z376mh2EZ4PJrBZjz20dl+bLqK@2?XA2f^_HGes;{PKiarkRbkWJHfr$Eu``Ti6eR1xI3T@Ko+_RI@UxHyh2`ThQHNiMceEiF+AK7$Rk{)$`^do$H!Zs7Z4E#!YNDq!UUor zVa$aYEPGr_Q!LLB*Ln`R4rnyPFkGJ%z{%-tJK$OYw4?NBymbP_ppBwbj0eEQbMiGI z@>3ci71h2A9Sp82bt`9XBq=Qo!5e~{<_jQxk=$BU2J_T~%{t|oqPL62OX`-iYr*kr z&eRAmf~>f>l!boNzw%k@7yJO5lY*o~x^PPk@cB)l+fI<3iL0QU8Cr6!h>OwPNAnqH zTnWCE`I;RpJj}0(QVf}Z@t2xcb}hDOjS!khoFYBF%28&J6M0qPJTcJRY zoLE9@J|H1g{H(y~2Od%~|3U$WcXt6~CB80o@t1Q*PB@wDri9z4vtrKsB@|`iuGXPK zTM)g$4gyRlF)0UB&H6xJ9A^SEY!aCyMKQcPVw;<{Ng7l58;{(Spz;|C$H0hQH`elV zcVC!MvOqa#agWb_%SI)}Tux23o@9m5A1XLjK}lAAM{d%uUGsIBK?TJP9_rf`mPn)Y zJL)(j|3chvmvPp-vk-5sB!-=rK;Q|Ge|G-yksv$e^Re}gFjosXk^PA=BeCok9-5If zW262Qs#P2`no8*!1AtB>D!oTIJu4cWbBKp9*Dvu1k&IdfQH66SpUY^cLPAJVUC-tM z1Qz{~Xo;6wuE6cvyF-Gp?kA{yCR5}37S276B+D!94Fj6{^CpzhM!_ky_Z7f&4F92> zkix&tEQfe!>0*nP7NZS~GfII3zeD9viww!Y>enIl(y>UA!r4I8r~XTQ8q(yrR?S1z z7fm@@mAirJi8>AuIC+w}&<@zgke~oiq~r9rtPp@%2+(4&!f=`<#rr96w(U)-{U+{) zx++cja;YWF2qAs*!CeAJBVvNoWW!e`e%?hi=qW+c%_9m7Njtoj78T>{A&+gWWX|9q&qRdwlzx{Gifb6k-8` z5T?|Vn?Vzy#O6!P1HA$ic^Ze>F(pMx#)onOr~(T^a6{*cJ|f{MyAE1zK)xr#b3{GZ z|L{G@Y~y?FnQ0&`oYUf&PC*_-H7A#Sqk%Nx5Su|`u+7nEmG*U<&O4u;cLLSawgsV- z*125kKs7Z%W$b0>6DmatJxRj4XdXAK0^P3VxMxT+tWiKzy+mb0J5C`b9s|^5DvzGw zK-xKKP{Dg8H%3)F1en&BN1~gITdnUrhZ>5aPGkBbgX9q~Cx;O^euYhfriUY)WQI3n zx4^A4<2jRJssywSG8e~ly*fE7A`TgsparJbv3SgUQ2>Uyj*=4>JSe6Xj~m4j-1 zg!l3(^Yc|(_nQq}<<10?KFacyP7PWd1#&NMmHymUp94yQCo7Hm*zHZmoWVizFRqEl zuB@@1l{aC`AC?ur%*OZZd`sHx&KWRg8cpr?&Deh-@L`{+7Oygi^_f~sm$Flsh2Apm zA1m1JNl3eh;*+0---;|~rBSjufz#oHMWlZuk(ro%O}H_z#=wxF4u|R=4qw5LiozV@ zQ;{l2cN7B%sp5%!pBc~uRBaFGJ64BaDGiP8u<=X(+&?cnYy4)B%umGsOVY8&0u$Szz5@m#15!ifv2v~#V=>b zPp_@S14?lug;zrZ9d;hR460N_c|ym4dW4a(VQjcVzDWjTdh(j&1Osm}A(bT@Z~`63 zBI1mM1bj4&Bd5Y?SALqCF|a4t6^7?J{B(vTva+?doPSzdt-FOH>VmEZrPM6{KG^rw ziECbQ1wmF$Kf$qU{3gLO;$&Bm7bH)-`PewS$f+?4KT%+nVV6A(U(=&R`^}U>XE0~q zKUYbr%aS1tEto;!%@Weix6q2r7OeJ@UPS|d8KXr*7sDF=ucWs5A0;)j|1PO@z_*`g z%ox}GE2*(T;5{4tS4qtYR8n)jcqE_!mDGOxE2(AUNpI(jyqXez)W#2MxVE@xRNwNRaJSpz&IHDel!2Bzzi4gi+b%nSkb3pu` zB{l8}-6eBSNsSn!c~TYMmI`r5uIDZoDk*VW4S51EMahLCG&?Y!#!lCAHe~A8_$(f@ z{YrICn{$xQ3I~s3jhX*fQak12SYr0jn)S23y?JW+D$ZUsPDV5=``%TnJYT7%{BDXhtx{I)TgcJA-yd7 z5*N8I0x(D|8|S%kzo{+4K+anbI1W>ONbELdi>ia^%U47{aJfyGQkNv1d0kE{fh12f z)TAfn>Cz79W#U#m@rG)O{Wf7eZoH5aZDdW7c4C`qo7X|G^F8rsQ7KH;2syY_SOTbH zuji@3gTrR>6Y!0#NCn}F9d=+4Z41WBY;=Wn>*JKh)aFr}+HW+vuzxwm_nV>QkR`S- zD(0Z?$|sMkB7G03_?6kvm88Z;HXPLD4_O|qLiZ_}O`bFNBI1MtD#_|}d8S&)bwMGu zR!~R{=MR}6Xn4uZql0ZshPK8F&P}Z5 zpW>+r_IJVe@YXuh3<{?hZ=aJAPR8U7ciKzYjWij+ckIdf*cch>1vjIO1+pSlL@$~x zx%j?D!}=xrk#YR3$ET-dW3`Iqs0J!$zip3;9Mo9eSl+Ne9kmQlM~&f!toH`@0%90r zo0VH_eEW#j)i4Hyv?rn)V~*8BwA^KY2^ulW4_IcRKj@Xh+3wsJA$FN1U5vEPdb{87 zXnemP4c77tvWSFoSYnr z^-2Y_I_cdGh)Kz@i8_@+#&?43uIv-j$U^>x&*f<<{oe(= z?+_D%WMM3O7#HWf(c*fN^`V@My8Zdk|0-%6Xm!+?z6fk}R`tkEQ2c+%+`z<15ruT! zRR3DT1_ZuWM*#yfj4FB!-0)*f{qK2M|$KAv{Hfy3tSKz~2d<)|so z6EyS%0QGu!41R3WHs;!S((`{*)CL@DH9!?L5~CC`7Z=R?PXAjWJjhw-{rt|ET09c; zu`25idReu1#%Ho;rbv3x@vRv-V6eIV{qJE1m#c&Y$*tkvMDb;)En!rgvk&M8@a`9udHNDeuA(Jcot>0b^G_=(`(u4---|5#MnN=NCKK!AKxK zptZMm-}P6otvFlX-$Btmavl}wy;f>8 zla?p5fh&~e{60A#39GK%95wH{(;OxiRvaKs{5>y=7@d*cUkujppt?CPrCdg`=2S&# z?p9?(E-pII??-q>HEN%?0Q0(!t)TD-JDYS7o3E>fhX*gO&x4)I-?L>g5k#6)wbyE8 zV_}p;BZaj|vDvAXyMQ@=k?&XD@uv!iqiKSz3KowmIVbeK#TpcfzwE@5++`7S#~ITl z_TL_=w_19(XFlvnwiDX!MDe5A++m0ycYBP5AWOTn4I!=GsTlA`d<2RaR`jP9F>w%+ zFV0l1%w0Oz&QxbkI4WZFH|ccxmq~d~VLsB)G$HNV2(V(E*py3H=ov^Y#)u~+%_A4% znws5iYzOcFiTNhXO)V1p2yh1`yS@UIsW^QS$_PT zMS#ug?`+3(Jktj_N69aP4bZ}!N=Z$AM;Z~Tql-0>L#x0Q^Wd8?vno#qj<{MQ9lRe& z+Wo)#W3B5P0#>4E!c{44x@?zP-Q(z#JGGIVhU5Mf&ob_3utNk2 z!#W4TfITX|Ep~?B2R1qM=I~I2-C=na?VEUGjO3v6Feq>3kYyE)Z6X)>c@P`CROi@M zodw7QptaxFAa&Z%ovOZWa2~cY8SXa*>1SSS!fa__9Qr>9PRknH+Yfiq%ll2YQH2_F z7-bvBrLCyLK^lXO9#9lLKa#xZA=u5vua=3Zv~M@p-?HJf9GPsTi|m|ChYR3GQr+ir-$JO?1p|$pLvYc z!0?11*K_>?kcMD!{c0}Y*&ptOsF<^eZn@2|Mss}P5aRplYx47}`ae76bqs{FjefQh z7r&`Zi8g{;53rq^HZAvGV^_;jJSKKLZ+TY2ufSb^4Wu3k>gu^S*W#t{8^KoU5oMj# z_;yq9CS2A)Qm*R?4RasN=&nMO)xuJjuwEo_NK-K5Kv+~m`oIFB)DHsY4MpRBOfA-@ zisxYcp`n1wNe=MtqTc`o&)iD&2v6qhLBX@tg!Wmo4o%WyyNfa08eT5tT4Hn&T*zPG z5^Z`)R7;LW*aep|QfS%aFxcMu%LoN}-Ns@a&*iwkrg8fK1)R@h<;dJfr z8PRaKiw!y&T3XH#rfb3WthI<9$*ZxMUz4aw@sv15cFJ{%+p{7&^m;X+KQW;Wyd#pS z!0BI1!p`aTy2|-1Pt!j)hh%k4KH$ic1f_;{c#+{QsUPgdjD?iXL(e|Dd?^8S6?_U9 z`Yl87WG(A+stk}_uvE~g*9FzA#a*sL3PGyE-6p62J&@cEWTi?m5z~^x zHOIwfHzey(z>e9%x*2ZvP@vH=C3I2>Z5Je&^OJeagL&PAZ!s*<5P#@SK1KiygQ>I4 ztJuZMJ9{sBFMPwQhr28|VBQAGDO~Sn)KjLQ(o{BA+o@qTgDkUt$|xn21bDJBR!7|b zVM9Gkd|&Sa+N+5K<~34UL!nIg5o!O02VU>ZiwObG_jcJn4+4Y*=x5_$zVR1>&l!o$ zszO#~+l>8bTH4Zx{&D?%ws#qf9;N%k**Wr^OR4B0JH=f=kpX+r_pHLXC0w=M0bgpw z2L4mU6;2EleuDNb&xYMf)|08X6+*@mKc5m~<9oS6zF&pIQ{Z3pm!6qnHREuiK$M3as2S>AxYEf3dvN)BJ6hMW2rrcW)LD4pBw`j?W3 z1&ggL5(n21#${{mQdzwm@9Bp8;A*ylq3}O!h75f2?jJV84Z>z-&93kmnERx@8xB!5 zec?Hn?2EUgDgGwf!R@;Xo#%M?>x}MOKbnd;JQA?fAp%Ler4lvbYH2Fow8isZ@l4$? zjzfNDazg57VI272g3Kf1;kou^_u15r>GHM4ui+$IGb0%Cqp`pCKU%EmgZ)&fYCM?1 zutQ%4lA$RX?5yB~zD%TQwW8lO_D3$oBDqJ{t>u(1x*9^qC#;Q_y$AT}w*=aq)C4Se zKM!X=2qZ8M7%{DOQC?#80hVPX5Jv6t34~g6TwV#BcdO+Ix$rI=cALk`N%cUm&=9u;A`4!MV7* zySqCVcXxMpcM0ynEw~4`&G+6~Z}qeX-Kz)xannw2_ zO&e?gQkluqjqUnNLUuA%KAyjYoa^g-ts5y?658vNn#GAGEe39YLZt8zXV#<~<~-)` z3H+XsHmaooC;{N4Ia%QH?Ok(0A+;W?A~A6f$u|+MY244;nXBA`@rDy$0`5kqskyWty)ADrqPzc89Vfm-mc4I3d@*2HN47ia{UJ zu9^%mx0!!Ga3$;YaC@DZbXzi<%CqFNioMXx^*OW4ot=df1iXv6*5j-;7k!IFQDAN-ASJT}Lxl|Y-$@8^`O(GGAd3ONA-wDrx7$y$#7-ff!@%$4MxYW=|ot$P+i;nQ!Gou*$ z>`T1&=J1|WXKuwG!o7czAst5z;ux$QhQnznBx#TPmsyR{3idv`Ub#qBIW9o|l6iQ9 z*&kCBb+@cCs!5Y}W>>@9pINV^R;;J_Uj1Tl`LxIG#6ii0`54d2)gQxwE!!vLgXb#; z+~L#VASIKOx}WT5Kj}2J8sOvNkJ8t1O!>pYbU?56eV@IvQd=WgR6~>hdS`#?9Bgv8 zKYM3x>v?hk9p@>LV$Jwr56GXrv)~%y-@=?k3M$!Lc>C57$#+6?l9Hb$Hek<%zQ{nC zLxS8Tv{z6?@zkLy8*Ku(ojUCpac|>}v?mh7)gVkybH0vyQOW!1@?Nfa{oxAiZV(0e1 zD8}H!Egs7EnM-2O_^rCXc+AN3O*8l+Fn63uDWp#AIZM92(#%Hteg_pE&*t(y!KT}H z8MUx#_nY5aWzf$wAm%jakEO1;P5q(&b`H|<{?pX_p5!%8(J5;hv#qLE*0j8b;C?z@ zkb9thmd^I0wDea8zU!{5!uKzRdbd^7KxXnDM8HDpg?L$`o&aYJz67a>Qbg6@ zxfmbqb$ov6pZj;4RL@`$TJH&x>KOD~EwbC?^u>Nfg7P?gmcW3h0adNoFvei@r_&!8GjO zKVh18-&W>^bLLN&rsaNF9pcL;Ow-ek6X2mB+T%;m!cKP@60p)myr7qsmgwoXws`pi z$uNK*#tP+gm|@{yz-M{|vUR(SqxN|>R#>h&SMd!JOup=r)D>9c;rsXC_x|_pa{=%q z-w-V7()sEuFv27YxLIHV3^(;=`JGb&=wJOs*8%DS=bs$SQEOtn$^=HHS&vaku-0gy zUYWgSQjb+1t{OpQi}GjiEZ1pSZZU3hxnmJ|ukU!_(Uz(eDb;fRFx6qZJCMB<#x6O1 zBnL;9K__3at4FLPN-Id?Hx(-@+{b;#=tSM92c#pvQt!hoahP$%l5eVYeSy{!sJ+ur zCsnmuWF0U~OV(7?`3R7yy(o;;{*~ht<=00SNfc#mO4UxBrYD`DiML~vg08#ZIP;a& zY%_w6*g6of_yrFgn=DyOAhi*Rt87neIw4gI$|$~!FYgxEHNF7yp|R=mZ$C<<2Uz+X zvMdV({s=>SefK3$&{{Y(0Ooms>5<{>F1z+R3ye4-5fDVF6+*nu@Sc(?^7=yTF`*yV@s%HpC?%epKG^%V zc@35Do$uw{FXzq1;^#ps)1HHoY-RJy-?x>WneD7D-v*9|-;yBqzD+Wn@Pc$yNq@OF z%A|u(`+k!SVVg7-+yq-`7y>Ioq$|qbi&*WRSYFk5)ujQ2E$vm~(jzhp@3Rvne~R?R zd*3rR!-y>Ccf25R)=Q2>5*Zz&U}i|Ho-K(KlTEbP^fB0E@`~g+@kAdfCrN$>(X_*~ zf08w3=wG$;d^!35O^$!dG#%WWFP{(E++|R^zjasJMm3p1v{(t(k37jk1-wZ)2!nhU@gdu^K!Jv}U>Xtu$mn zVZ^)^^4|r4Wvr0-+C$X2mR&`aJnQLNPELE=b@t@SPIn4MjM{S~%h+nwpVqYP>RplN zUu(1jA0LmHmIm9v%YxyKRulIEMHLKH=eXW)ZwTI(M_{q-Hr`h^leeqJ!NEb#x2w-2 zWFh#8pTG4EyzYi$O7hjlcdhdGKUlC0uc6+dNL5cF`0g8+TlYplW$t|WX)iPE6RX)4 zAFBL{w`Pxp4g>%nE8!;_gp=FCMceQ>a+;72 zOw);`r+5<6%?h0%dF_w=mCQ~(H*PoFWSVNC)oU^AallEJlG;wODlQAlyXQd9UbX@RH5wv6+LCJM zzg53Hvx{mr3+QCGwh0Z*4%e$hSH}_kVolF_s2f-i8jU6SyZV2TOaYuLD6P*Yva_82 znzUwmUr~aku*i8@;Q}iUdi$eZ|HWa4y?+$Gxe?`dc7w}G@~!y4S`?K#MFxdF1~SoU zz@YVXbLT3;QLN~5&@C|ger@>;Vs1zVr>$onJng8*nNfH+kLCN#ju0y$%N1klJIsGe z43}l$=@e?E=~*x1vV*T#hd5;wY9Y2gpA}C!J&CAMh{Xt0PAl%rPe=A_o)0Nx@Nq$o)3FniHno#MF@z~_*PFgXGi^ZDvk3}AEY@m^Spw2dFLc^*0yAg01#{7CGKcc8q ziPR?J_p@Ohgvp~ZE2D*z0cBafInz4=wVvJIc%P5e{%f9THq=zzY$HXPM*!?Y-Z)N( zmJWLh2M0&r6rOI^5>gk%q#n^zK;7h%498qIsqNo<_OyK2^t6e3c248@$HIB?`1k|3 zGm>9%pTCmx4HLbykJqXvzj(9$(F&5J=v0#1f4S z>o(?B-mhvw%Ajb)n_M{kkSd$-=8?;5pl29MlX;j12!C7^aK_NkghqGLI&n<5o7pyu z5T)Ku(}g;~bS%K|fE+KUk^WanD~*snVhsXx=rm7gLTI%*+Mpz5T;4dLu`d5kAlPniWv3ocr{+ueCdzeF=?m5!+>bjoGZ=!4!PxUlWBCMY z(DfBZIAz(3)iv;Xm=U@7eM+m#GUhALmr9RI8}{mJRB3?Ly2d0%3A;1C$qo!JPa;=f zm}7yo24gI}B49szrqR-i-$zvr*y-BP{@MNGYQIrL-U8SFNw@LmvOv**Hx8q0BIEZp zNSBGwy}^-jl+-KhnnnhlbcSQyW$)fF7n45U#Cw=na{V=@B%xJA$u)q&<;hX9V2Ct2 z=A$+n;#}H{<{pzcAxp@BAIrUe<8JoYM2*QVtSg{_=0IROE0LO^4O@H21mz=Y*Q@_o zAd2u!-G{mRBm!I}@S?i@^J|O+H76mIv6sSE-)t|1Am;A-9GFAdHy*k@gsW5VqwAR8 z9O=waR+pF8KsPaCLvLf3oS3 zy#V-|M3C9Hhr1mC?=JAwVdHP=%fB9@6_jNiJJh3=A7|%r6*#x0eC`EI7Y_`3j@(2+ zuO*hSmEr_dKGVSQao*bH;!w}UjwY5+&>U++lu8tP%-$qW+PqklQ}-(1xLd>_DKYLL zg2B@Xaclrm~fIB z+EH}a6cFZ*pBRVSJkyT2nB&!MyR9lF$ioJP^KcS3ls6yvY4XH6Jl+hwAdmQ|E;sQG z=7@+6Rj#Ki+2&jsZg7FV=)3d9$e_)qq0O}zt4mDD+@+0>seliN>Pe)RYOtbDm{IDt zeVIE4^_n<=Sj>9RAtEY3TY+;Wzmip9J&Y(hzC^~D-GN5Yl=B7$v-q&Ip^maQ!;=kI z7mP*&r%IZ_H6}>KjI$4~AuI+~A!2^5LsMVK#hho*6}apT|uB zG}68q)hBYP$GLI&50E;jHq-?pq(m=fmdxXpxnS>wHA{&=mzTjE(3S*OoM4hFr;Je? zRY4$xRu%CSrfSTC9O_)-R2oj^BM&1n5~pR(`vE;!hsKP^8pM+wiHbj8Cd>9ci0B6Z z-xP1N1IK?7d{2>J`<)qXL??Az=rU&v_v|>YH33L{JVB+Afy!xZ6!HBDxcP|(Ru9_U z{j>pg-GQ)*pSk4m1g;X}N`azD(jxMQcaUFI*cxS%Lb|X`Eyiv^PbyvNjI#wbuZ47g zT4Y)|qY)*gI-u(?DLGmIIzsIJZ?}MB0w>`RTsDOXpWFL#70}US%Hur^<|kaE06$W? z-UV2B<}(z|9rIy1t`bD}{twhxCt5KMWFC+Few)Xzr~*P=^gyfIRjjPwhP=zm`^B^p z6Fh;7Kmj7-z75oY8)pKZgqkp$754r6kFD_(h%Z&}8<%GHeqH<*&{att*YH>sABXLU zBg+^Rc9G#I%r!fJLyd72=}dyDh;hEZn$Agq;Yl$^bE(mfiY{&+!1-ps=J~Z2`5~CT zp?zeVG!t91+*PrBA@U1;{Z1YynH>(CoDvcnsn*Ishz%NI4%K%)-zhlOk+LXlnKuKD+hc6g!YVf>xOiVZNUD44)~(wjiZG z(M-TWfeD?iE>UYiTCp5ZS2=a&`f^QxMp_0ZC4m9Du(uomV z@6sQ_(9AIT68}&DlJ28c`;De(9n0UFP@R>`3`Y~X*e(Z#%W#WU6j=t+;6~6a$P~`y zVf#!2v&hsIILobL9@jJYRx%5>W;*P<*>` z8{I3v-@pyg#!{degsB)gFbYetsKKYu*fn(o)2)y|2)(GF5n&3R@pCF3z-qiLXUmRb% z>HDArV<^#Yv$|cle?Yiwvw&Z(-|B|OqGs{u@K|o&6F#G74QF#oD!s{CErb6B7H-#`1|cMl$7zZ@CboX2-Q@Ps5J3Td69xYHUi1L3RnWKvqCD#)w?5F*EOK& zvcltfjhv%JhgTGqOwPwhUAUc{Bi6n9+k{3YZ@HlUX~=9H9*B}r2B~L#X+UvWsV_hs zn8(Kv>R$*C7*+Rd7wpK1SCldTtT(Bvp=TJMLQ^~2|Cc;tEUJ!&`@(#b!W+p7OOTLt z@FVy+HM2Md!j8PSfJAg^sqqIDhw=SC^@v6o`^WTZ93@(I`1q6%2BHQ0nh9-(~y5vHwk8oD$WbO*-U7tV;6CR>?P_>5>Ohq5wq)B=D!f~;Ro zQhl3jH1M-{7Uh&`nt<+baFt>K0cF}R2;vOh5>I?Y`rhs7dSB6loQ|MaB{n*wW{kfi zjKH%dx`WC+wJvonSmoLOG6@p0{fzG7ig_5Gz+iL0`|O=Hn`Sxjt~|hzs8{z# zHM~-1qD3v}E0%>MQsS2S~8cGyyfhVBOjc!DDI`9?)x2 zQ{&B#zgkn!a0-o*EH{B6=J%S^=xQuE9UDR@Y4ahdKj{BKQY3nCeWM^+%I>^MKtLe= zlBX%9YDlpP;lg$ZZ2Hjw1LF~8Jmr$(nK{tAK|JM$Tw#wz?8|w-=>dI6Y6j!Yu}b`1 z_;tZrsj+O-+&O`dH5aNdxGM^3jnu$KFhWM=um<{yLJKwdGk3P8F^8S0V`0eLs`*ed zYUe_;Lz};r!h!&eYJgdIW9+3d_eYAFk-nHG%+4qVqhK#>W_IU%Ws5wk;o1>8eUjC= z%oi60lynl)hOwZ+1QqU?>aGD<18H( zHk}P3PZ35@7>|(kC=G@FYu{BDvLc9A=I_v7HYa=H@k6O>@m5>OiEXs3g+8KK`b$a1 zvyac%fhs>Y$^L7d%_r#+Z?u%2G}*KjTBo-Z z&zc#*)F{-=!iUqRl_p&!YA6#9QSU(;xmjs+SCCaXlx1i%f0ym`;i~9L(~M={fcXwr z{J`*u!Lkucm0Ms1;9=IzJ+fUm4#_bqTy-UdQYd*L7xPJVqD!IA8h&?GL_4KN|9LUX z)|5R7ErAVAnFI(7%!}L8jR-U;v5z#e)b!c+*0haBV#v9|E7Gp|v!7ybCl3o~6jP7(iJpmi_q zJ&2t8-(dAn+TN$TiyDfHjf~!nt9^hZ!G;BS9&#aS>YR=ZJ+O_FVNJLb!;}^KDJu<7 z0Oip39;ryM42KCl*^nCZg~jm(O{V$SDo27+Uc7xRpNbQuX;M%`k7+=ujsAPrK->{A zYIr_IMo_;x1A-Hp>m$Y6s-D;8EpNm;1PFje2Ps9K^K>tgh4P0OAwVv?d5j}7@cc=* zsiWOE3J){PPtb!i_?p5y(1(CZQ8EhFH9nMr?)kUB`!<&m<=zsMlO4&?65?`ywcs~? zQDi~b;Cw0>2KW!e{;Rm$!yEARV($}7fr}c2hxYA%0L<{+>%ZQQ-QE9x5-|E6;@yr3 zjU?Mb<`?%hhL9X_h9o?QQL1kAtL*iQha`+oy4KmVTqFcSX*!2AyY^FIL0 z|GNMTlGOhIF#iL<{C@(#00+TVxBUMHfWi2`0Wke{W^oU>mxqmox0~+2N=0ZP{sQjt z#qug5LbLc~w&arAUSgl#w2<;{=QjSExasY)PSkL5jV?ABGIy8eysBy8Ku?w^8b_Dx zq5x7+q4M*h8?GWKjZ%zPZmoFyRZ4E8Sn^}SCho@vX`+rI93_mi*2p{+ydY)q`acou z&R-&g$yPkF;$@!tFGLePdkF05b#?gY;M5CkNa7wfwFr_Mg%9S^2)$u71e}t`lIGD9 z8}GzvN0QusJ1Bwu2CY{sw2T^n!NA_3V#o6chmooxp(RTrLthAb)l~xvvZtE!dyz*_ z$|94p2=Bb@xwH>_WpGeUH>-E`_~K!r^=L97X&vvC!upLV8It}R{ZUT+9;9~gikLPF zp8d}r#7zxXVa;zzR*m0mch|&gDsz;u@xQQ@%w5|}n6BEZR(^_OW7D)t_ zhB+JWyQ{RxaVMym=wpm$g?oIPw<>4e2-M!?i?t9VNRC{Hm0}TlTYz3A=wshh7o?9N z^_kNc@ht=!3QSYxF!E$AOz4;y@w_FwSDDPpLZ>ab306$zPQq>IN3L31RsWT`Zl6(Q ztMA<+|-D7 zubof^q(h;qNpZhBnb|Z*0g?3f^GxikF zlVX{pLv3&}@Fy~`uyyEAvkzD-^l%t}_;A6uNn*o`ewo>~doZ3=_-!S{W3yzdrW_Th zDZld=>+)3hJ^ysgf@pYt8K!5eWs`evL3f!po4ygLvl7QfkiMue|33OljF8IFHQNYZ zITq#fxTZKBT2o4HEr3AmMK~Pt%MB72&V49fo*lb} zDztLk7E~QqtwKM>1%}>H`NPUth$){-(-Q;}_|6Ph40`|V&aAg_nW>m&_Z&m-1PG^ME?M#izB8-6casm+d@(8rH|?T)QUte3h?e7BT5YNu|ge3gg?+iYDZ z%XkJs*_Fgcu3lyeTbq2USy9usj(>1Q0+RioV@&ziobFxZ)qB-CYT%f37bK?r|2<49 zBNa2Uolo)sQI+b*!h4+mUTlXLYgB}QfpRobU#Tutny{!Q2hqj|QN1^q=xC|c`x!7| z`^6@STk&3|JAb?B_EcgS!Kj`Cos3srFO>cMdHc$9TG_nw$Q{7Sub){~SnR4*f7AN+ z>4nJ_aw^etQZ_6;WwET{xStg3`P0JcRAIxE4fu4zU;`cL2;XH|+7N=9gDr>D`xQ5E zRbpB*zNyaVFZag$uy6Ex@&+$@)#<`{-zWfOdTFEu1QWJm-u#+1FtA-+G8+sTZ>oNDt6KWMy$3fi8uC+GXC}iY-IJU#p+mGx=hGTe zWSR^460rQQoct2zkrk(i7 ziQMYl7cNUJ+rpmtmGW!atLC9%YBZRSL|{wa+jx$v7BIAlp0~}uR(`r;@+_JL2AmL_ z;Hr41l>mpIbgx&vyxt-3p?cWoo>Ak}&(evojisH22lsa7!K*FJzICJfE7yltmbioG zO`b|R#YME-&{X7dO4R^akw*!8X!`6QgVZ8^{5ol*-SQ0_j9`s5yt5N?PHXLrQjT(% znQV4*<-E&|Vnf9rqXylBD}iYFbYu6ZGou))W-}w4T84+-sI2Hpljg#oRh`xN|D-aB zUj5s7mNZW7z!>QRKL+?CuAt|~uJGEuV4c-JscNiM$jz8L-8??ZHqNh?)h`8UrK4{g zGw|QbI2BgoFLEO6;(ia?@*xQCKk?|goe7E9xE*HEaj$F}ORBB7r(gf`+_GC7HnaCt zA(Mb~X~$r~T@alhV%PP}xkI;ajM@>7uJK&>P0`3n0P69b_*tkIdT{b`XEw_xOo@M$ z_5(?s3HLpNGOCe2sFWV(*5r}EiF2~Q!x&qS)5b>NP}g~!ArV;UwuDVv-TVxx>)CKZ z^^)vqDmQsblf)3V^>cen4=1Y1vxu^rtw%!1g7|YFl36sT&8PHsIvA!bVH}KM8^Y%i zuU4M5oQnJcmVe#J;0jPX1fQauz4RiZ{F8`#-SP1z6bBc2%!F8PEmWpQYasfk7CqqZ zw78EGEltAwqqe?5cX5AjW$Z&OnEjl!n~9A_lH8b-#rK$S2xq^)_B$MMqSGWw8x!a< zo+$=|A$B8{UfFdnlHn`;QAW0~NqXMIO<;U*p-U(hh9!XgU7l=`I&og}87+Nsl+_f$ zGv{35_`d9!^`$MQxv9?=KQ?Mc(*Itb|NfRee1 zP>N+~TK4pdM$r9WV<;6cz+S7X=zD8ue_YR_r=c7`$o%aA%0n*U{kdBFSV+NPf!sW9 zkJNPX)ESeY6Nkno+Fab)0}1QD&+&*f0z2hJdi!u**9??mMC>|(PWTuY%3Az4S>kGE zfr@ZpuI)vvW+NYkaL(+N>fD*=Cwh8L<<(%SQ+kUBRpERrcKbTMeY#c)KDW#D>3e`i`gYF?$a`SaddVhcfu{^QEp!SNPOI`DQTUpAufH{b0Drm8%~+jiUft9jQ}W zfb)*3GlGR0bG7gJCR@$58lEOZE(d|PSz2#jbrs(oqR_ zbxxOB|AA)BWjuHFbnd5m)Q$Xa2wO}c)*SWSeW=vYFCKLzF^aav>td&+s1p}`Ni^B4+`Z%V}Kq>oI_-X-L zdTISXtYjZtg<_mBzY*v*5H`zRgjI6_50swE@WyBCqDkXf$rXJ4oE#tj+{ImO=X!q} zJ|8F6PKREfT#a9W?+)x_GD&)a(cnaaLb%2XEH&e4ay%A=HqtMa+Y*B9FI)~d>4YPTvn zZ(v$=8xC7`zQh?4hw-Wy2%dj63qOmk8}HI(m^-7l@;TYYx5Mz_{+)&~hO5dNuqhc; zW;5}3#JLPdE_&P1JoTf#g*w{zN0@&LbX)ay1;~`QAI&|A~wUW1XZSqY9EjFz%CXLC`eP-%=Mt;_sHl8;AK~nMIh;YG z*!!(Ktp%&8QTWsa)ZrLa!{p-S6`q}J)rf9D_4AD@qx@gN*aZC)h~(uOMkItueXcyC zuA0uxai%$S%l5WA`;7`1chMw`s!ah#(p}u~!+$`5iM#!L&|f}&6lR3hydV}Ol+4f| zgeSSRi7NG9ZDJDx+yrw%;KLVH`e1tlaJ?*+F=|3R{=B`9caf|BKlc`_qQd0T;36?D-m9y6bT=PtheVouA}8qh1`*CR%-?63f1#$0Bxn>~kCJ6TwwtonoX$ylw=Zlhd~VNKV=0 zPnz^_c|}p6=erwuqQxtZMaUmWk(ZWG|E;Z{{^5+R^neUfn$LK7d zx9up{DW{TW_wudRZ_(}=+`l^yc4Ex6>j!h(oL{ZNf68T0d<(LkReSwv|J;n;H!|%7 zkHFms$fhTGyTH{^8{t!CF3?pW?cjvs=l1+o`%f+tarXO88F-_y@}FFWu_3-*(%!6h zf;08)U;dL{%pH*|kR^(7p-R8?|1dC(;ET1p(K|n)TmnbwNf1C|4cYB`4E(-FmZ0i# zR%q&d;7wm*vKJTD+J3MC5QOT`Ghc&Iiv6!hAMVFK9gNVz@)K)J7XhdNfxf&mRRo;^ z%fR~H?=O^wI=4)IDMZ8pBK%!_AHsHyA3tMbpa;(ZQj#1A_~s}})S4h78TKvRHZML@qMALIu0JpY^ zd<0=Rupb|6ZD3qNdnCMPhZJKEp%wCdqCb95fpmnXXy<&-e{}a7M_vSYxP1TJNijv) z@{Hcz`Ps9&yZ+(>2VsXv#6A5tC&nG_=DX5Zf>M3LJJgug^LbYUhJ^qlXy%Z)n4 z@wUFFuK3e2!-z2AiT&G_0Y_KCp-0!J3_0x_UR-857)cFpL8TWay`jw?qFYB^35c#u zkN&>!E43+${Temi==SrSgDW*)-HCB!y0Ze-y!1$b#BN4%3Qo?w5GnlfSDp4N4E5%0J|qou~5zYjch?0+3~dSrDOD2tc{dP+lxqyd z0+4p~2a%`FZL77s($SPUB(b7ugBI#KW<>)$iQUuzrC|MyweBsaA-uDDTH|P7R9{yj zUOh{+xaWmN{8|zgnhdZgys8nOkXQ}}FAq(+6C%WC{3oY4On85PR36cNkkHr2$(OWz z0nNb%L$we;dRLa?vm1tQzck2eq_I-$`#}kwzgacEHeU+A)v;TY5d+u)w1<;^0kVaS zC(zr6vH<%1@yX6+_vmz*EXd233DQ`6n)TWah7BGTd~E}%jae7NUl5FeZY-9Re&1bq zuY$zd_K2l>l$ktA$OdoAOg%9itSL^tn~*c;zvwY|%Czq-5E77@MD*Rcx{leEkoubL zgDp|n-lqTb_iTl z^>Z%XTM{a^n9`7jXP>Om2qq`}W!UfF{mO9NWh7trtMEvwT_J}*S$e^HZ6Sw0gtW;2 zQL6u0C9`!8w^u^;1fyNyC<|atVx;GYl_JT;MMnu_79m^=$&f}57LU46T~KV<}thbrWL5;1OoL&fDB z!0nANO3q+j_?3118Nu%gO=Uy zQhrXn82ieqmhqi1-4X+>swT5P*&`mN*#Qz+zG6kbxN)3Ly^6kyEK}iW7Ug(gFkE3d zD$YGAPhjjG`lLIREX}n*L_axh!mklA*KsAuslf|b0(f2i&x6e7c$YtI2NR5LdS9kK ze1FTIhV4X%QV_nxzh;UH47E@pD%AgNn4!Y1BJauyOI^2xi2s|2Z-CsGB*Jz}^LtYq zU&foy6KPY9F@d)v~IXE8@?#kW9dzoH%g*yl8g$PsEspcEpOe() zz+wyT%c^X=)6l2GBDs2ASF7>Z)2V3#h#kwJ5TNWTw^=EQ#yOiP*4*zQ^*bC3ZPaQn zChNO7mK_@ze}KktJvr$`ye0jm7`4=hbU({sj9RMwAvY8;b#KM=kmWvvzpB5F#9MI} zU>k``6(b(+Hsz)+{7*kuI|-HE3B3YN`;D!nx_9`SU+XJ2D~?5kk*F=K_D08YB?vf* zh>oA<{CSon8>Wgs7{Wanzhf`{aWPjvF<+ zO8pRwn;W<2*o3m@*YXZ)0$`DRSS*8LCL<2Vm^+X^OrOZAyfZ7ye8~7Z@~7aV!6mmE zrZ8v5S2Ai1T;}U%X*+Dm6$}`!|FE77kWqv)n{XMmsUueNFtk0(pDp* z3pLFRs4`FRgu{UAK;S*!<3pvl{I}XK7n;ZJ-@V^)uPc8gdm#}ASl;;rA_ATyLa}2I z7t!O!jJ$hWxOvPfHU4m6AcADkJxN-yq@dg;jp4EHHJO@dC6_0Q6TZwIN^VCH+r~Ck zi^%3BmBC{slZO;6Nkx*%l0_u!F12Kf-2Sc9xaLh3j-+crh*qkBynHRbTcJxv5cHiA zgy~V|OeszPFm-2Aiz(xGzK=wJM#f7BV8#kfLlZbk>w7dj3Mg&A#v7L|M?1wHckq)O zRIs@#su7v|G)&0QT@Ps63s#L9aCEj&Vss_ApZ0JfQCpIb4ysJ1T~N2|4Dc&vY2Og!28x8a-yY&MYaC&>~Of(0|e z_v0pwu?vs*A*GW)G+YTZSBrId1hLpH*SO|i0Za@yfDAakS~2)!%^Vranj@U06pDHk znm(1rbohRq@7McHNS!-SlIhk>NaERx_g{oq*Tl7$rNF3dRg`m>ealLpn?+2PW119A z(W_1;3mc@#UnlPGOdKku1i``RBukBAxN=wM~VhI;mmMf8`b?zOyU*dI9jrfnpJak-Xb zR?A3@)sk4xeVspe7Pp~aA?QKq)yEb?kx*5vEq_i}qbx(chM-}ye`YKo7;ijcX15@@ zCH?r_a{ppI)%O7R2>g-&eV26WgKjkugd^6ae_@MMsqPZYhOCNF$sTmt$|#YHtvoq@ zI6_R8-?U9J2HgxpPQx}$Y9F)*SnmndiEX1fI?Ya9s1voE&2OTzP$!kz1JVyN(2{10 z8HT>MhqgGi@1E@KIJWOeaIVAJ4nK#fO^mp!jR0_vSEmQ$zkal+Ls3t~bP)Bw&_!*Qh}}GO^aHKDKkpiTWNdv4ANwdeEYT zcKl!UAYBlYiCtiKd`uf`?DBxG*aAD20}}+d%Zu zUJgd1a1t?cs~&U3Aa_84KJc-DSx}sT-?04Ad-`r{Xq=jZ=N{@f!1cTN34*u=d@4nd z`tv+K6eP2=uoKhsgoy#ez)Vjt(Lbv=tsE+5T)_rXLoworp}QR4@H-NGW0E?!daM^y zW&3yCa5aupYlon{YZPjfFw~x4g}T~(a(D~0e&mHZV`x8I)L9<0DbcPB{mUOiT^aq{ wZGe-$u(2<&0ZjMC<6`}}B+u_Rch4SgyuKeH0$>3M2r!tE^v4lIv;@Te1@qP7NdN!< literal 0 HcmV?d00001 diff --git a/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/Chart.yaml b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/Chart.yaml new file mode 100644 index 0000000000..11419cb2dd --- /dev/null +++ b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/Chart.yaml @@ -0,0 +1,10 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/namespace: cattle-monitoring-system + catalog.cattle.io/release-name: rancher-monitoring-crd +apiVersion: v2 +description: Installs the CRDs for rancher-monitoring. +name: rancher-monitoring-crd +type: application +version: 104.1.2-rc.1+up57.0.3 diff --git a/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/README.md b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/README.md new file mode 100644 index 0000000000..e0b63e0268 --- /dev/null +++ b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/README.md @@ -0,0 +1,24 @@ +# rancher-monitoring-crd +A Rancher chart that installs the CRDs used by rancher-monitoring. + +## How does this chart work? + +This chart marshalls all of the CRD files placed in the `crd-manifest` directory into a ConfigMap that is installed onto a cluster alongside relevant RBAC (ServiceAccount, ClusterRoleBinding, ClusterRole, and PodSecurityPolicy). + +Once the relevant dependent resourcees are installed / upgraded / rolled back, this chart executes a post-install / post-upgrade / post-rollback Job that: +- Patches any existing versions of the CRDs contained within the `crd-manifest` on the cluster to set `spec.preserveUnknownFields=false`; this step is required since, based on [Kubernetes docs](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#field-pruning) and a [known workaround](https://github.com/kubernetes-sigs/controller-tools/issues/476#issuecomment-691519936), such CRDs cannot be upgraded normally from `apiextensions.k8s.io/v1beta1` to `apiextensions.k8s.io/v1`. +- Runs a `kubectl apply` on the CRDs that are contained within the crd-manifest ConfigMap to upgrade CRDs in the cluster + +On an uninstall, this chart executes a separate post-delete Job that: +- Patches any existing versions of the CRDs contained within `crd-manifest` on the cluster to set `metadata.finalizers=[]` +- Runs a `kubectl delete` on the CRDs that are contained within the crd-manifest ConfigMap to clean up the CRDs from the cluster + +Note: If the relevant CRDs already existed in the cluster at the time of install, this chart will absorb ownership of the lifecycle of those CRDs; therefore, on a `helm uninstall`, those CRDs will also be removed from the cluster alongside this chart. + +## Why can't we just place the CRDs in the templates/ directory of the main chart? + +In Helm today, you cannot declare a CRD and declare a resource of that CRD's kind in templates/ without encountering a failure on render. + +## [Helm 3] Why can't we just place the CRDs in the crds/ directory of the main chart? + +The Helm 3 `crds/` directory only supports the installation of CRDs, but does not support the upgrade and removal of CRDs, unlike what this chart facilitiates. \ No newline at end of file diff --git a/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/files/crd-manifest.tgz b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/files/crd-manifest.tgz new file mode 100644 index 0000000000000000000000000000000000000000..82436d2b13f5b29df0246117f3b8b76bcc8c43ca GIT binary patch literal 308599 zcmb5V19T;8*EJg3wr$%+$96ilZ9D1Mw$riEVaM*+w$b6;ectol@tu3e|BdtSF{<|7 zsT#A^uB=saK65<}VKfxbU$4*ecWd{(mPM+#CszURDKcwMv&BUIq&1bivdJxc1?{|v zp>elFe~tNJBxvb!=^$_+&xNErpU18KSQ7F?AQMX?^GCk~4;v@vy_TzG-V9@gA*Ftm zn@-;!H-OjD$jITRfp5j9|JyF?oj+)=_s5WifxqY5(2{|l?_1~IQJ(+rw`cd`@ouGs z02!$Xe|MiRTykT9MI|mCpyq!U%hSYdb~YE-`pU3e!F8K)tin%$t5_b&QEmIo*0JS3xR$} z$$NF%q%4{}M%A4ogu+Zlr-VvN?n*0G{6yJX`$RDwi^G@}Y%dY_3er~>X?RP7Qojkw zPN1eQFRp$jqeLmw+)YwI>KFe(rrwKjng>;}1qvd}U93zqVea9Wk1jThdWplc$qVnbS_ktg?r|y< zsM`ATlJO>yHLHXzqm6Q-MXgq`d^6KT>2y>Vg5^C2pQ3RpFC!n39gKW|sf>aUc#ps;;*3ERi>BtS!2T5@ zTRX3n^V#?nt;ZN}l3^D|F_ZYp)ze2N$h zmHtT*dG2v`dF5jAqR@p;DHfTihZD?2vNGkG*IrtZV(ltsuL`Zg)mF6(Bk4J29>y8j z?4)yY43jI^d^7Esg~2d-;iRlOK^wE08^{qQ?_btQ__`@%^|0mBymI#|Op;;ct<+2B z70{?GBd}v_-j$Iphm1JWSnr;eQ>7Sk;ke)GzS?!iXeQ7jGmOWe_)gIsWw69u9?GypeEMj2?ze{rOj>+))w3p?>(EO12>a@jrq=SiwV7OTSKN1{ zr!Z`E(c0LBpYl-BMY7R`{h2(`29(k%DZw~nVuRV)g)m}~X$1F5c_@%`RG0MLhrKkj zbJr-xj_gR2B-`}d?p+Eu9_}E8EN;D%n;I#}xqri+NyC31tW1`zMEj+PR^+sCNu5VI zHG=e=g5FFr?1Z4%ZF06ObubUz7oY8cu?VTq((fBYP*b zX-3uME>YM>fOKvmdUOA@d)TK6KE@uJWWFL%iNDQDW2sh>Gi}8O%sDPCMB>+F;Grvyj-tbP?azDIeq2ydiS)D6fy zoB~VGWqCGk-0#w)z`B9FkMb{(n2f_CZqZvPfim(aiHRaN+u%FLi4#A)@7dJ)4J@Be zSP|8E#KCDL$QTbA{Qc143&+AC1w6`W%s(Xc2WAjbKS&|#@5oI*rh7#a^&win-=%an@TWf#UK3>$yE z$YDfjGZ~z!Trn*JdtNrUTt$%4SWs;zFvsW|W3w)DXt8r{G9Hp|nIdPIdeW?Y^KQ2R zj{R;b(gc%)5Sc26%tB=&)D&P*|1H2Gk`i5P{2DCCV#T9ap$U%U*k27g#5yzSP6VZ= zPpf?8)9j)<&NP>iW`p>%Yg%&v=CP-^991Ny@4r*KCj zD4h2i78n!4q3|t;fKyV&R;1TQR7z^X^!~VRL8mwH1~Q`jF!-G#zC`sxWjf5-+527O z)#viU0wP<67vzd=EyVz30(T@4m^vhP@7f?d!429$g}JGl~Px{puXD$^Rx^;2+FW7T}+!;NsHJ+i|< zohDI)yadNpRJj2}Pfa{3O#^=<>i6Ef&)%}F?CkVBfoqdbZ2}T`6cNl~>kM9g(TBS=FINWoAZ>mNh?T5b9OG83 zPj7R$KFS9b8kAv`Bs$bXCdVQ^*18r{UoTK(=D5U03Z+T9k(M+o;d*ZuDZWic&>{Jm z=~0@4CfNQaTHU!TMuU;#Vi|iPXv$lme4Pe1;K~qml==R0bT^yu7>nLPLM8kp)gwrSO8)b9#E&D) zW5#8`iG@5Ht%241wj~ouA4yvMi1Mqrhzxbya?mw6xx1u$%+{O{M@Ny90Lz*X-U+v5MmJ_F<$9RCc8qxji3X zjp$AApr{l=?hNtll*eP|;KRr%FZtom?KB0>2C*^Mw1EXT#qo*=b31s1IX_DKUT3Y0 zW1V#4s_9bc%0~OLtaNbPP=Ba=dinZ(O^R7!(RCj*b|62Bng6sXPRQD8`puAunq7Oe zLlew2XjH{of!AhWAfrHh#GTaNR;dVAc46+E=Anca8*|Kl470BZyVU`$}CW%Rwh*TFqaU4f!0yy zhsyTkz&gK#*9Qv}nNNr$bRkH^<$iS3EW02b+%h9kNZ=1f&PPZ;ZwD<2w*kcoRf@aj z6S;jKS`CFYgWLh(F=|pzmV~ted|VSw-*(%^+_seg0c#Jo1@}4&zR`Uh)!Mt%%u=yR zF2ECIX3n~0R15;go)w7~QIy6siSeVEQBY`_c{`CNlBkT;2uUa(OLRQ3i?lQ`H;JnH zl1o($=G^b^g%ZYjY9mIsJ4g+duqs<4wJ`ebOC!~H{VO>+)r?KzNQLdIQ|(puaxi*- zq&iwHCHAHECsjR?dkkdmPzzAIBjXm^t*3gW)OKzo-#88arBQ-tO`3&V7S!HYzI*@= z@aZ5@01vXzD5NSSjj+cvwC>)}CP!g;|ndlV{%qi~Odtt~AvDy3n6sBF4nBnr(FGL=fI z>G@@1adMR$HZ^gNZSTg+74A3&iWQ4;-{vhVTL#nK(bX2|<#l-2?*a8#>cBe^r@<@v zMOoMTM&!UbWjPi$au(<)Q04Yyy2Nfk)5zu55nY9ZxD`1f>vnOn}q_gg};dVN5$8$*{D{D zHi*20C?iCo{Ax_{D9(qt#C34WEneAUIaQS+5R?cfe$LjhM7OkP&8oZ6Mz!cECBH|t z)PJ2t@QH#E%hx+Kq#rWbAh*D-UcFChoaTpz^39JEep zf0uf-+-&*!|K^Fml4QuTi}@~h`u-!<^PzMRx^Il`u56|+faqk5Z*HLZVGlGM&*k8k z@O@?Y24iiW`n~^jE3^dOdU_h8LERO3^V`m^v`2VO3YGDN?-_6mxW`0R9LBqtQ`C5G z7Wo(gl^PalI|+%6PqJ4zN=-XQtt{hRb%>rsYG2t2Sdg7Atb(i0mNc@`GBQ2;JL3qp z>E*{|Lk+fAC@yKzbaC_x$6x57wk%3nz(hEEg+^g)^S?o?f9&LewF-I!@$9tjK!ddU zGpcPr`isF~01g06e2|(xZa-d?p8n{3m)5jgcP&|?FlL`X7>&2hT{5We1f^A(8`M-T zLR6syf{QZ*3N{8O3g+1PI!eUEBuhmKq<6Q@=#j0p*FcJwnl$^Vkm{aH|Um_KjVkZo_ku3U^5&hxuYKg?Xq`}59dTduNRlZIW>~&z}n3U$fpzJ_b6iV=2pK)06ec$%= zM%a^?ntlK`IZ=w?aD!^D(_)lx0gmf)E>+YATfB*`;oGge_Vm#Hynd+)e^zu`#K5k_ z+x@YuwM!WFY!8HX6HfAR1|}NgD3BHqEDn_JBXB|UM2|GrUK%YP~S1~8|Iv$*QXe!~P{4-!N%AV>O zU+W`3yv<7I^*)sjQI7uATuQxUn(v!)He|=dQlyIwZv+9FKR@*tZOIXZ7n)hF@d&c8 zX1NaQU;#Hv@V!=U=LH-urPiV2tNr(cRa5GW2jVYjmje5XS!b|kq#N-53GI?FcJWV3 zXPkA-X*2X)J4};zy{L2Ns|LM2QDlnKvkzyq;lxWkvpN*E!9~4H$t-mYDoiIYhWV95 z;{9nQWF1K+!YF&DiExgIw1Y2Mmon#ZU5Vd*2GTf9l&rR{(WQ7P=-_(Q9%MIKaY(v5 zXHHCR-qNEVYCn!Kd$H%n0&JF)0BlM+K~H8IEpR(=>CKV2FWtAq^3vP%BRjZpyezXK zb}W|{7<=lRSfut=!{jKCl*p3%ZTy*Z0rZJs#OjFkFT%^RBG(AYg94`x?^DV}I1@~T zi~B`}Veu&7Hg2E4zaWaoAU``rT0#*AXTo0sy0<=Y!TG+aN!B&l$Pz#-As zA@hSkv%ooc!SaJ36>pK$u5Il)FZN1xCbY%Z9ilrnO0R0b>85i#S%uen`~q5h2>@Hx zbwblcbbBO*H!OYj61aU;pJzh-3KfN&t@A1zDBWE18b{&36Mc**X;}exD$2Xz}brHjEhf*x!;vV3bA} z6O7t3;v)uJZXpfydY444{LkoL-!RjXU0G_PM0ql~2c;!Y<2x5YugA@6PN#zgJb$Z` zwn+#4X{C5+lJ6mkSO~Z6Ws2?QsD&FI{8Nxmy0+Uk6tE-|mq;J-T;V<_Pu9cUUs|C^ zR_6cJ3eNt4Fv*6(5=}_ONUC$-*hP6T_w%UqYydeBkzl0<^EK^ER{0J%_B`f`W6CIM zXo{iyvw}kZ^<^0emGv{Y~-KOZJfr(GAxE=*&%U;BwS_c{q@ARN!PE zA=g5f4vBLn8_XV#p0IT;@oDZCBP~hR=_&y9HsDP9{4xy)l<$CyHXAg z^*2KghCQB!;v@v^?%IQcXBvI|S~y7&y_CQ~?+dkon!vL-Za(IfbdqRGRTbY zJI*&Rv5&u~g2BU?D_%)01uYg9#z89< zh6gSd23Ncl=Jg2Cz>&~V#YNB!ref&KgE0?8+-zjMg|v*sTVZ{MM+TXWu*9a5);H7i zlF18ogDPdYv#FMsx%ciQKlp20=9N^^VMWxL_Rh_^10xi_y=80~Ob_lmxB`oM?g;We z5PibGKnaC9j?9YB5Vm)U3>HdPB#Q;BJn0T|&O4?qsAdDT513k;NO(5YR?q)Om`d z%m$Syf-I&dvR~TKYst-FGhfnQn|8g{aCxdlNqmxpF$=W!{UuWcEtEIN8Derb6Aucg zy+7ebD<`_xmyCVR12-RBNj9t?XM=a`S;V=&->-{(>z{Bw=OTfEeFzA97LYWH&n+Y2 z_w31|Yz73D=cq`jAk;wOeaN;Z*#Ves3X<~YzD+F;&w6yhK!*$nZ(vZKQ5_`niyL0qDJus>2=_-BPUbkh&O z6X;KPn`H&?>Wo7F!JB~?huwaJ{G(_AI@B|$?_x_9z3rV4IT*avp9`q*Q!0bM?Y*)OuWA{bqS91G9TApmYQAe{VvE407R!7UZzA^zH-2LOJ7xIlWAyc~0Lm1}{H^rmd` zY<8w{>+$cS5vmO1I8zd-fT(~m$g9C=Yl!**Y zUL$6m%P+jgSem=oJC@{WYfTHCii~+`PVM2jUhJ0vrCy*aT1D7fjaB;mN}ZNfX(pP3 zc=^^B{R1L+(p6^VHBuOzRuU%b!OXg2ęT{>=-Qjyk_i_^32QYPcXnWV%^>?|?P8Ln5?W>t zKsh!A@2=(m^3rq zI{L;CURuq-BJxO1LGh>|+1klMj8*9W3C|Y?5DUoPc-_@qKz9BB5~Q?1B}k=&TL0)) z4P5&K)V9NGI4INWx8#-Jwce~s$g9V2eRX2hAOPt>{>+E>&}4yvivw`wwg+Gs3K!=9 zWcHcB;?2bM=P>{Q`4{@%WeBK6V66N!wf*1{T4ev+J?xu>ADrLSL6=-D@?ipLb!$}W zU0TD`e_Kvp+c=)O{XQ{kEco%XSEXR!0=LPYfn+8`GMH}_f9%O2QMgMdWvOY+R(e)c zRdbe}(J$%|NUn#p5kzO9!s9Ef&_AUEH_4qL@vTPw`S-`|!>2@ht*u5<#CPSGM4gf& zaCoMop^K>zat{>#YTDcd^8E)X3b6{Woo;!$oLHKRPW)eSRW?g_N9^==R|5_%N)fNzGG=ht7HP4lr~)$~Vt*cV~Dgt=;9pAyHSZR$XxwZ{aQaM660fEcesh#B;t9(!JhYN7oxClMJ4oVk0I zP~bRO^}|kW33z6Bud%y^J>50PZ|NNvzrOrlLgsmV#RVy&j&Z50r=-1n-Ou>O%08$f z3RDAT(+YG;6iZitxm^-LmDfrUz!xFRX9;uzwrhzm#78WQLDoS2mf~y5$p1*QB|ZQ@ zWn_n9#eHU{MdQ{5YUMUN(S+jcTG@JPYPGc?Njs0B6`b?s%siapJ~#Hu1*Y5l2phRf zWW>5yZ7WAh-4d@=w|V_EJkvQQ93R-eue9d`(#P1P;&2qUc0*L_vA8?icoUBVtpAxn z&7cDbH#=rJqLP}4SVV^fU;n7>k4u)ssUU{X@FyPn`s5Hy%HuqRypMN}`*8oakCodM z&WCCD*u44S>AT@+&pq?mw{DNeN%Kz4wDt+(wxszTowJ>eFDii_Uhjv^D4M2W74F8K z{5A#s&|a92AomO*o1q9qlMr(@M3ZJvM3WSN64!(3hUEkT90rs)to1)i9Jaun_#Y*% z*9LN|rwMw`a1ME_cMN>2*9JY)`5iako%kQuJ|dtHCtAEc{5e1{_b7eG*dH47sHNbODhO&os$%%Z}m5^*1#q# zJoYZ1gztN;Y^0nKQQCTe1l%*EkY4*uKwkU#nHpkFPFdG-jkPk5y)1-?=8?y`9sbX8 zs!1tDg6~S4)X!EvZ?tR>Jd06Z0FI5+gewatinE&rQFgr%MRsC?taH)sifi$zqm>K1 zEu0;ak45?%{)0sJN6A!qXNFKH_KAEL8Pt|1uaG>7wzB^>lPaOp7FHP_@M8jCk@0!d z7M-W-ZTw?s9YQwVl*95B|KowfAMmS@pfmD9yjqYm@`}M}Vz-tfTCZx!^w`a5N5uA3 z*YXd>-(ZFjyxt?>C|lRqUOJT-FAK)E7NhcaF#KPBygxk6qc&z8`f<&wN>*rUS!dEi zG;T0!^jF7s=!BU|qGV5yzN~=j1#jrqs>ta(w=`<%C#MTCX{A$=u zjC&A?MNQ2zZ63RUl8sDipZhXYc6cXihGc3WPFzlnO?kCD@&Zr1i4%#+?`FNGR;C@ zRe3t(@Qre!Vu7PvDpa%%?)!8KhWSX%ha5Lc6>9pQWoqb_h-4@**9P9;T~x;mHq zlZ?*Thf;;PnY9zw#g?6=&H{U&-<5#Wljx9i^|8%sRD}%klp6XJ$_3K^9RMa`?G9bJ>tow5 zzL-~yYh>7^H(8bLE$A9sKy>t~bHin&+Bb44EoYK+wm%v?fG{}y5NBxn369yWo$S*W zty^Gs(46A!^sBsV{V*BFlxjrr%fvE_YCae&OJOKcXnrEYPpo&<{JdO?g5UE=LNTh6 zw<(Tl1}ig!Ge4&$YNAm?)t;>1Po%Ln-`DX)Eq2j@q%KO-$$nLB+LaRl59p{UYiG`4 z-bWjkD$|vSpV2I4gSSVmWKT~2Butw=?5AP7803WhNM@U zl>#a<&1ul(0(S&sF|vzZsh{@jr>|y3a4a@Eqd|}UmTU)v>Gy{c-+|cC1r~N2iXzw~ zu}QNpZ~fe%!G!P&{gMwb7c6|rPLF;k^c?{lZ*S^m8BY(yh{ZG(!=@A>wR*PkuJ12a zcPuKb3LZ~1n~k^{EH_D);|EZ&!bfky2_`a)@9A4I&6g)n}(Zt$mNjC z1z)+eII_N2g0NH`2n+8A5X10|SJ`nw7gZzJyS*3~z5dDX52@IN?YbfFNS&#zChe|?s_te<-ODIF3mqAA>WT3G z*S(^6ifWIPM*rCyv?{gbt_-|_HYYyPPLb5vUG)o?Z$z;C+}yo#S5eLi2W$BYmD`Jm zvL!U!H%}ALZ>4`Xk_4t^cq9b1A4l41A*nJd5_-fu`e`EDh9X>&1g5c}^^o)uU)KUy z9t)t{IU=3(50dn3_X1FMnA7#=jX2@K`lA$lzxSktE?1r%tTj*62sBex9vrMy#fPPp z3F=o!J}uuiAhqn6_Of6u-#BFek!E^NO%BJCW95QMGkmyaAb-I4@1^Yh<1!c5c{S z--E9g{TsrSIQYQ|MX|gGGhdM(tT{#e(V!6Pruud~a!{zrX*DA3j z6p2)Gz3y%+^sk%J&09`WFt;sZlp&Ieo&}pCp?>hQmU7JKROHxtNb<@}`8~La8uqnF z^fOgAcXd6A-b5qHOQt#vFeTMo4z7udb_M+lop;O}548fTmufb!H{961lQNmv`aNnd zN6nkkWoh>AENR~=?)h@)dm{304ror!?nkOX18a|~`x2%D~KD}xm#ZE1tvz^El75lmgYpYsBxmL!BXjC6cc`;)7Q#%7ngZG`@ z9-ohJO#kOw#OYoE&qh2;_eAq1fi(3I8!x67J?&WN^=1ug$eatZ(VsDUK3R`$OOG4P z1n$0|(Uwl`8zk0k^Euvb{xz5DWGjB|-CHZhot<8_yxyBTiE;P|#>3_}v6tD!>ydlAp6X8UG%iiJzT&02k&m3(!O zbv1(PYMGb-)uEwkE%D%B_a@x(IJ1bxGil1V9z(H?1Kl9@Xc1~-87OAa@-IfB)l4_vNn}1AOVrn?Jtv6V`uy zX##+Ye0)Fv2%cJKlcLmAHMXMgaBYh z|6}Y$9Lb+<4uXQT#hK;=xaYte77xP$FO_+P9-c-JXK^z$b>+Io{0sbHD|IKWp9k`D z{!x9eey?YDbbE)n^)liNsvBgi00vt^Orx^R{K+2V`)#N4dBLJ&EN}kONT2F2i{U4m zMPfu-oQ0j22tKytKx&T_PXLCokBZg~3;+`Cp?!6YsXvDQb;W*#zF&M!_VI*87p zW{F?P;R9k22l=f6Z-3?{L}PZodH$y#Qo;51S3l&y&UhU+FfP`W1YZZH5ORnzL2?Db z%dSxJ>mj!rp3+6aK+H5!qq7Ob$vpdR4As^HxUv*Ynz?E+w*HIl^usw&5p^WZG;N6KY3>JWU{bdlE%J~v&V?8P7#mt(7zw+OXaah`6|T?L_AA7edo+p#6$rdI z2M|uwJwH6W_&9SWqd0vZ0+oT2So*1HMDgC>G-aPWua74+!&p zr%-VHeLW=H^$C^Xu4%a6EFn%*(?3iaKv+Ve$N*Gd96&4} zf8)W6|9c-cjO1UNG=HK0!=x#Y0+=-K!_BQ&dOG>1{$C1sy@Dh>%QTdd>8hbyJRm=eLR!n(XUIGNQ(1jW9gny6pPxx8j~Y&!$Y zJd&$%I82I~GQHm`jl^ymm|l<@+Lu2=y0%JH#=o5o@w-_5b`)1M2*+s3$LcO7!FIC8 zdbwOG5LcL*>S*+sU*%}~hCRUk!*hcLXcnNcMCZVE6QdmaZ;OWRNXJur_)dZv!R4R} zRg&i_lo;gbt(#+2RPtD>WrZV4$hx?MKNrM&ot<3RKHkkc4XL~S8t*{ply1>Rf(p_bOUvRw*pNokhWHyW{#g%)`+NNJ8vkef^BS+{ z-VOh^zfZKkPzVNq06MJ&z+cS+4q@j13GxaM*(La&CO6@~7!oY#Ar_#(Auw?PG=Ja! zuKE8$5C4za0!S{HJ|Nd6Qo@NMTBWtsMXAKDiSKy1iQBikbb^A>M7wVjH=|}bMxhcp z0`f82R6gbs#15v>;h3~s?MDNube}*aWAA)lO_pk&0yh?*@>tzELO07 z&=_1s4-079Se23FQ3s_(j)NRDxMnTA?10Gi&EU%RUNz7(j@?vLe5ycJVt1FVOX?vV zg|w_#xu|5}O)J)bFwis=zy7sXOvM>f7S%d+4QN>SOBjn`{Uwi+Oa8P&u{<0Zffy0; z!N!!me}*+ID5iR0<Pd9fFC#{BSv;wmUQ# z)g1KB0o2pRfblo=EDinXg)9x_)4BeGdH@i>$t3`I0QJzhG6DjnlHCATOkMrg)I%kU z|F5a%4?6t+q@FQlNrCjerx%Xy@PSrXmAxmnIww`t-ysgZvx-IE(_Whu>d4kp79V0p zQ$q=g2#<<|M#7=YGejfUk7OHY$WN|KNb56?#jo5Fv}VTtq`phugW^TSz1!qGz)d<8KF_{KZ(rblX4P;v;b6K1u!ZPBNB>^Ltn5J%c-u112JhOO*1Lkm)_|S1{ z6UxfTZ5{`BT8h|r;|s_%0mF&FV0VUmUVbOn5+1Kin3n?%G%l?uzf@6`Ng$ja@Xr|n zAnX_1q=L*WFGd#|JTuyNG|qedL#TCl=si=_PFnGBFX-8+-JQ;SXq*?o?nt{Z50s5aRDNKMiAWQ&2c6ok)cLKnc z1qAQg0gUq^BuZ#;JP)pD0SaT=w{ZrLNbYq7GZFxaWbtRhCsh;j-+3C!@h4A1IRF>@ zN1n3XYXSci9w1K(6d3=;o0%E;*LiAjjQX$h^iQ;)|1(c|})=tD0Qu({N+^Hpo z$NKv(6g60gnqq#-!|bmj>w1r$fbV*X;d_yF_UU?&*=>I(?;Ot6O*8zm1>5f)aJo5( z9=5zHC3_kt27A|CU<_(^%r&`+en{@@WsyT`lBIQ^NpW|1!u%3wh_3K6P*ZJY zh8(m;;~hyU^3o#J5}!&=ItPuL6nsb14JH}G^@5Fp4oN~~EELbwM3_bN?NEY19mkR6 zBuIBYI~z0cTHO9*mt%Ux2Uba3!A$3CSB1&hH2<3Tbh5N#qBG6;TU{n5_Gt&=kzrx6 zyC9;yYRT?=;ptbyiAdv#T~>?rMIvPWUh}cI2o6OcZFI(S11&3&^z6)7(neO*F^7w1rRGjrFr14rcL1lzTVN&|XpI ztny~dF*xSfG#S)#^TG3qkLO5AuJJ2DrDXio=)&x_9W>mdQnjh$zh;%te4gd1j;8S2 z0#*c{X*)%+Rm|u_-P0?HKvZ7+&l8zJLxPBw#bO(um#25k3-Qgx%`E0 z<4!ud-UHr~?Hp2CvcyRvB!^|Ab)joc$60X2c(s%_>dg6$fe%nT)aJEYnjCzce{+RGII(4OwMIWilb?*n)z5sTCAj0Q^Ki~4FIh1$&@Rf6J zb4mN=mK`j-oZR+g3=zshJ8bDqUqSEn+VzgmGUh{=AOY{REBS*xW5ebd!Z$rCUM1x=kg+B;>3 zQi`DFsirxwK9r8<^BO1PW_FY5cdEJCX_%lwo=TjTUkv-bJhDe=BRo?)cZE8ULXua{ zyO-Zx6hx)sC{bLVMaetl^iy?VuOB=Ig?Qxn-~bzCG!e{69H2nc+<0&ZEG34t=QNW zRp@?Hbtz@38m^j5#&6ko1$XYzdpApfg{hK**Q>SW63`0YoYWOZ(X#LHLmN;OzL9$t}Yos1lmu~d$M z!vts5{$|=|m7i^jy;FouJskBUPm@^^V!eMk54hmn^vK9d%IeAbrHr1Sb?FYOb#mcj z9}nbb0H}O%Zo%Ods4DE?lgO@?|PNtWnM|kS)#i`U`W-D~G0vTJ+(FPjEQ$`!G zf%7vN{4oZsCeOEsg>BbRjwrrt3-^m=rVl&QE(|getC)FjA$tKfOq%Fcil(}eZf5~Qr*z+m)AnoDGm3gTIEw%0;mN0c zHGQt&2Qt2HVhB3)q?Fh7IpO0@vqC%M3nkB+bD5nczd_X}!9DW`<}fs6COMV0gj9B# zZE+(9k7Bxw^SNckZfw)hz{=O{UlW9v9=yUMlR?A}6yP-0sk;ftTM@7FY|5 zC>+bZFbl=?4w63g0bfN>G1C}N)>H9ZQAM_9TVU>(qZTMc-q%H9ctDwidC9|ZzaSs| zPKLXI%;rt={ecFQ<@0Tuz&e#Jt zU4Qq>e{JF&=1*GOkUpz_6NfCBud;?AD>}!Z&ks#$a!yg5IInCB@EU$t8_Bbm6dn0N&b80`%yCgBc?I zWC<0TFV_;5XEI~%EcN(V+3SZAOCUv@e>Bal;6c-d?5AVsMVb#FHwTsMH>JhH8%(Ir&QWC@p;%a>uVXe7Sn3S)5@`G|#>{2rm1WyfLgRN9@;TplgXfL}5t1FN1b zJ!w3cHdF;UKTY1wk;v&7X{BAh?c>R%MwvStr5{vhW_Z@0@Ef(kU0Utv#T)E+8d2Ot z&igHdmuH~x>kLZ@CePM|a_j--XJ&jsd7r9Ul=^g!y{zz1XFLLu84}yzF}4FCy2zO) zT))GhvWX7s=ep2H&*BwE$%UK%V!<8N4Pn|VYH#Q(Z}LXNFDmm&QuZXeC&sOB;Jh~3 zMnhMd$In4Swwb5YCWY%FH-ixj5NF~D5iXMy`>_9P3nRs*)Br3t+M zhpl%E5-nERM#r{o&mMbbkL^9SZQHhO+qP}nw(UFTyzjkL_p7fem2`KKUip!gwbJQ+ z0!l0#wfe1nQ_@zpZQq&FglUTQ&X;;5Ut%g_AVDeCEWeNy^c4VS&4m;ckBW#f6CwDR zEdJD^a#Qv%d=Qg(pC;iw$KVmY?8OLVd<9WSvRM^|>6#{GzmeWJAicLZif;JyZ=)wq z{e3r&uOdZ8h8leoTZo*sDgp!NNt z=ns;c%Z(E_|1lHi6Ntr&yUn``x~kDF!q$`-V@-{tDY)`jSbT+QV1v40mJU;Y5~nO~ z)G%bqUSJ``(z$glA0Ma4WE}}pdT!<|CVvr~fkjk)D%GOt#yODPQt68ejXjDB5Rr}C zaHKr7z)relAb2kDnnr&vupb-M&-)BrnBqPw>^AKeuIQ4kibHN=` zwa3i4WCymk`b{Op(8>7tp~Y2t!`e6wI+K5POg29Lhx0|^9gbv{+5KBYZkp9`p`+j^ z95#z*d_YWE8s5EiNM(!9$D=UVHL6j4XA2x{BM|zO?LU>wZb(+D-mq!?nJ^*M=FKT) za;M7KS_Nb`{NA&CO-lWsD4J4S7PvxpfCk){B2Gfp(+CV?d4*|vVM>}pgNbmU$#HE*roV2=#Uv!j}ImY3qAIBrOZk*`7I8aK%pa#pZQGc@fY|*SAzM?R% z=bS?NCC{%$canj0e{Mm{GQkVITi98BeC{*2xWz{4j^JnQ<B+1O`d?LwOs zG_-vzxZYxXu-Y?l0897Uvz1e@47~pkT)QpS3xb-!j=5NT;8*~&en8+LoUx#@eth6S zxTdFLA$h=Fv8^Hy9N%GhR-9bi*={qn`{0NC8ugA9lJumK;MctvCyxChn3A?OMz~j4 zDxsE*BDxh@a9t= z;aGvTwoX?&>Oxu0 zirL-7Uq>y4gNqx*xI_+(^-Qk?ZCV?}qoCAT3M^>Adc1drsAn#gHebBZt%f)OJE8=I zH2Qw3A#~EQikqBNbkyN3P6Vv>oEuwAl5x{}hAm0JuS?H3Xj1zT59LFh zC@hin1uvM|qYJg-WoMjg_czXR&O`l%S2xH+uwGpYy?5H0+QAS_`1k2|lp>2Nx-`9i zlt9KYWYK?rg(S-5)PxJJKEnmV)T`^OaCA?3q(KaD^-O;a%%JTnUHWmzbB}pX9>YX& z&7YJ9CLktIMAL@*BqSp}4pljRJ6{pWFBW-qf772wpR$A6dSED@waOr+?*{)kZRTuDYT@ChRm0uMc4V3*^ zhW&XMumgW#n-AQx0~?QVr#fTziqTfTGoP$uaSvXjob%_OPJ}tT;)RF0yWr_g$Q>{C zR%N)z&>E^-ati1cn`WOm{<>I`szuboYcHQkrqk@opz`)1kya0TbANge-goA%KrT6L zPyU+N+nB487(Lb2!`aW5&&>Hhl<9gfJ%mVqT@;XlPk=rO*#D&oFpdEH<^IDcViRCA zb&TAgN2;fB%!$}9>^H8h7hAw4Gefo~(4$S%w0UnNB<{p`G>iH2u$nw;C}_<$D& zsbcdJo|O*QOCZy#VKtAbEA^|7^*qtUzv~a0y96?9cW~@g-O1~Dc%Ds|cTzz;87Z%N zAp41-lCx=RdsIhum5fwJbuYOG$@&=pwFjiq&<#Re1gP}bq2pf0K|5tw*jb2jiqt$( zS|F5umoF>6kU|;UB64vna$pNjs@2x6=ErYsO;fK`8Qt*K3pn?UUHzMy22gBhn5{N$ zfvR;IUx(GzMD2v8u?X?bOd!N6LWAc-(lCloUOT5sCZuD+^DpZfL3h7L4O$hF$rus! z5#LEcOApxauwkE&I%@}voG#O)MV#N-O66?8rOQBc_sgc@&!DloT(|4tUv8l6iu$nT z&m`k4fR?WmdR4fyR6VPuvP*rhFH48nzn0g416~T^A8evR%q9DD5j_4fL$L7m&m}KY zMm)S#>W3ZD-Cg^q;=b5b<+ZuCKI2tob#3l7%NW`L6($@w(C-DfkpgiJZ^IvzOJfk+ z1z4V|o=e@TVtYQ20)+)3#=V=)?@n%ouKqp?>g$Xmv+hMfNf2q+(<~gir~pObzWcRT zz)67O=Rn{Hl4K{4?TL|Omk-Rn0q#b$VnNrJ#SP;8jJ5~~cbdv4T}#eIpNfowzN+mO zvATp`=&zt@?EVN#lm&&i8?htF`5SXMS1>=@6m%er>10451_!o-WEiv10p!p*4fV3WrL^ptON2rwGqB|6 zVHhU5bGbHAmF=~C>UR0Z@c<3pc22ZcxWoFu)y{cq`EG0Qj%e$!$y@teczw(w#I+3l zK(yDQz~ct@ z3B%o_AkFkF2V}=9>Pd0Jml)Yr#oz!h82!rY#th_ox7Gqc&xOf3eg}VcvpClA1B`77 zcW2?KIK;VAms!||)aoNTRECWkgLnU%t;*=s(x1t<@klSy?6*`H!!`9kF!Nt%F`tDj z@hQqhhI8y+(=UbK+=Br`!M^~Y-~k&N$$iB@!G{K>;8_O#=Lr)Y6VwenGy=p6I5hGX zU})seXLCdhKY8ZJk~$#go+q;x>VLE-sscLpyG34N{feY;usQbtcC^7y1A((_e``Ss z2r!KPyR2a05&nk(?W3UMqlfaBg9Re|Wl9lTNBb!L-`0-(elqN{kl?>#`ul+B+;6t^ zbY>O8#5@|D8)+KCuH2GYn#~R@rm|(9@4gQ+U3rm= zyn$>2VD{lIr6`*TSn1S)a~cNBZI??P4W3I5hthh1i-OV`0Oay-w;DaSSBsv{uY=wf z$R&FuuXhWK&yS13cIDR(;AD^0(bpO-raA-oanSGg2>{%72#fGe1!)xs*?(`MZH)rvn2P!Aj5OUOwR3J zd9YMa$30J4PG8(+r!!W2St&vds*#4(1(-3EAXlxVdy5)4J|v?a4iEA_ic5Te zSFO&`SRxa1j0g)#EK5z>cHt#v6kruFVQ1&&bGL1oWS}U?qQ}O$h>MGQImD{M z*=buJRX;9;!r1+@ZpRIg6KN$0ARD=HhH+-DZi5FOnLAPMbeT)PMPiAPg_Uhvp=@;u zOXFkH^-V%8%?Z|lx%$|Q1)jO6e<*JpcyzVaa z&wPmsH&@*ZV5uCPuSpET;hfZ57yT|S6`Tg-poHtzr zp)LdS_w0VHvB_eTkf!^U$` zj-*;TDD)qk$CZl<=tM)stT6zn{?dQG)Ep+6-z6q5Pe_jIp72wUGLSkON~~T!2;}tv z>O~PJNfiwb{{=W`?)b>5%ME)s=L@E5>I9^tpSmR>{zX!IA_es#EHL`_CwhcV`_o&s zHpGOFOcNJT+)j*2%0eW{&M1jn%K? za`0Bnrlvsl=?93{{p0lTyuBuzQZ%8*y2wU8H~8pU7GQ8I#=nWjGfj4i`H z7n*2gbdc*~aEP-}wgjzI(vgjvl}vQ=PBFZD#9rNReOo50g@{$Z5P6spDHaqj+a4p? z8Y79zHK)GqYwnP&QTbS0p}wUpT?9QnNO@;XqLzSI9YP+0>dZfKP8Rh2EDVj0m%D6% z2l!-k@}LJuJdI6`<9vi3{Yd?7d3J z=VY=xC0VZ->9eyYICYus$mh3a_O@$fv}$Hw{bsJP!Wx2qJZrZ(~6pn-0o`6e7ZfZgxrWBFjc)|Vbs7vTD@M< zXKLCyiOnIRMU9JIsd7joa!5pf8OHt4?%h8s8|_G^S+G)<3Pa})3|8B<|Xm{2~_p{rhdyP8*7qJ9@g9BydTy4^eZ;rIg z0&+ph)(x85Q6s8L6PTOfdX6HI$N;S{y3{Dq^uw0G3b+{k&C{9i6_AT(11d|NLkvb) zZ$6cbkFR4T50&xm>*Quov=o+RZ#m3XZ-vw0TBRD zSmGP;gqq2}=S#8JJ9957HNda-gY%(*NrVdxtlAIj(pm#Wgrbus3tttTT=YS^Afe>GUh>W{Xbi!UI}@)~f^Xea2Cosgft zU2DGNeOsLv!nptGJz_6c(M7j@_{DXa_zytdYhcwT{mpp*CSTYOks6t23OTD-o441J z7k8f#!}a(;B(ckfWRQyl@}IDdNWzq1lj@5LI*5`5qtr~aKspG~F(@NnQ9zv0EdV&f zTg$%!GhpnoeSVj~wgmqJGPBDEmwU+184DTXgwxYVNewHk$!Hi+gF*V&NH1e*6{+BB zy9yMw6ZsdKX6=^V5V&+V*Z%`R(N?#i*Ej?~@fJV=FCR(w8=TVyob~Ub(yTbAz;%Q7&DPQ0Of7DK5T)vqx9RSCt(Af z^rw|J10Pr+KJ+e>m$&f8oxOYTSV%bD(6+A{I%@-aba#=rjqwv#(ceKHUayH9)pm5sFaoH%!v>_Tr5lI&#`g|_yJRI%>?WJmrN5|jJ;=xm_}Kj!;3W`|dU=cD-h zt@_Sk{nc&zmBZ%qPkImfOt;(fnip&GsI7fRlVyjzO@QtNFoNaA6gEWjvdS~Tst_&s zJt(I=h~sn@O`^}!@q51Aojy9t$&8a50_Fe!Gip146{#tMAbO4L%+9s`3#B`)KUF8( z{KNZ}&h*YXl0rHo1tBPDVQ!!zbZ%hcG0U9N$1=y%#KZzy0h@^qPy`?v@SkGPUVY|> zH=fd0dxQjzNCitr^uJrB*R@+F-F z_AB?Jk1gwUwPZt#*lefRQF6*7H`mu6#e zVmV2hOm~IhI0E7GlR1(R<_$Hup*9x`Y(cYTOSu|+h`@zJWJ0(s3tOak)_}RS4QiUr zq{Lo|f_U4^ya(uznHu5ClW8^q^P4Rnb(G}cOpv5nyM|mBAGsss{;9`Qo!!j^bkMJJ zw!NiY?wHcLO`T~lR?sW2T%BQSoUQyk8Efa!W$l}Aa*$t>w2d8bdwDVEuNR1VVD)c! zFns9rQ7pP;5mnI2dh-Iq;#s@3pND9Jl!Jl1R2mkZ2ph{i_C}tX9qlyLS_KI8Rsfm< z=bW%4LQQ>l#?`jh#ac!DVbQfEZf!XoHKAIyh{-&%(+K6uw^xTd_R$Nj&em$*?}wss z6-ilbx$Ru_6?&F+Vmf-RWnFs~H<`7AX1he%Wp=8t^h1ncOdt=5x_7;6M=M{~8ACsb z$1=kao-iQv4Z4YCz~*ugBg3xPxYQVfh{#LCI5(^<%kQkm0@-EYI1lGWtF^w_-Vm_E z)$&rmg5AKj@Q!VO8Yt=FT3#QR?m(hGQz@uZ9-E1pM z2jVX_6~7|4**iSTWO`yErbf9LQb;>*DUrgy8agsJF)6uu*E}+f6dWA$bO5jrG$Evn z$Jvb`ngn2SU`{bKsWAu#3m(XhuC{Z}aed9`2V((b(llXS1zNia7nm@#=qN^)m7@R^ zhsNw;yBm~p?u3mF&(FZakARQQH~eE`B^jo4c3X^N2L8GD$8z|CYmR)@U$#C{La3*J)jag1)D)#; zE01%Dk8eQ^muZZLy6tuIg-Nu6NGO{5^q!l0GpDh8ZMu~_ZcP1XCKe-DKWpGW*#Vw? ziRK?rSs<8J($)h*J$T#PF&?nPIX6(SM&QAH@d^o5z*NEDpXpD zVij;~Q%;y)E*Cn^367rEB?32@-aYmTJ6gKwXV4o;aaho0zy=Y~cSt8|QLNEPET$ao z?=8}E$_oALS#nPnb0?dM9UOY3T3N&=s5MtMnuQtU%`Oo;j-G>!N(P>1&h8c@Y*AiS z#?n>5Rj@+zF1a?S-Z^FN0-N8fs+u%as+&Bvhw3k`0&5l1>6=LXnQgHba98_chUZ9TqX* zifywiZ5%Zn$6%f`YYr8iwk2C$SM~?CZQ(uoH^PW`aSugd~-nGR_MVMetWN9R-eE`PGKxqEk%!A-jcD^EA?- z|4iR2e*7iGbvCRSHNVyf)P`)UVOXHZ`}rZTuo12Y`SEf&7yl5 zzrR1I%~DZEkkis#HldxAfo5Dn9jxTVkZwE-DRS)hD3E2FNx5v!`#B8{klZW@<1T1R z`)!gEBKYKKUx^A35t4Rm6?-9gus-(n3o_q;QAC!<;`3~bzKPT%oMgah@>G+jFM%wX z!M&(kdVaY7iHJo`H~(h}TD8WckbCqueLD|_g&=)lMTbE8->byEhiMI{q5=fe17%x)^buq>k_|JQ4>3&X&be- z@w7P(w)K_SCy zxL9<+lmFEYg{FQe)>AwsTv&iNJy z+sk~)8qS7x;NiYM{u9Qx(AURnE%5w$?lpB3rP3B8M4lM~os2YuCt!PyE<~*(-V+`; zSdVnTfqeg&|7|ih{}4gY8FkVzpRi0Aomay?h$cGUt#%Se<`AE;2%2T%<7Xr`Az|o` z>XIo@#xx&G@>eB&Opm(q#qiFv_taW|L?NA1za;3FkLZU-FwSq3me|*>AF5m-+s&m*%HH579U)DHBhjOJ+Xf!ydW!mN8 zR|wlLV$|Yl%{xnZQ_UR*+;1#^`wSC9|0Ru@(GLIngEpRv;4v2`-=6CaY9&}0`Qic? z0+6=dnvU=ri>3)^^N9L{&Y9{$Km(g z+S=OL*^7_c!%|4a3h#Y`>`sKUL%`3w<2$#TkJOkvy{UR`v+$=vUNqt~7tQ(ZgBpz2cH2&9 zoWW(CvLykyyH_3Ju3gTbT8H`N5a^@yezgvn%rHidzADiD)&|pj_$G)1*eG)8tmzR; znV(Ca(Jm=Y0uKDFRz+^O{57s^uv;)n6A(%gXqlY7pr7(`MO(LdFX`Kn$W_?Nwef&> zuUvu+855wr2s#lOh0@|{`5)=RGm$^WG0z5VHdCyPG7-HI&@Ck@GEue&T+Q6CKfqtj z+sJV!=FisO(`Xw#ckqYA9K{XTWy47g_)IFHM`D^8D%BZxcA)m?9ij@&+5Ca3h8ch!=`tSc zg6D2GuQ-=b?!;gTe^h}+0z!bGtJ-Vga3lBK(&W>Na=1_4sz1)uq}`G53XVSc{s(Js-Kb9GWrb38X(&jp za*f&m3#G19N#fa{S0qYBYpZU)WiHt|7~?Sh;4) zA97&P5$sJqOB98tj#BUEN=XBlpyzWg1EYzGsTq_DwRp3fA3qdDR5jaK-$r@_OL?-4DPDVo?k`wq4 z%UC5-M?~JwAj0*AwBz76mJCzY>RVtc$Xs7(t=SseVQsMs){LZqaF(v|%5#};Wicxss z7yGx|3J1>ZE6yS!bS$4LTFWXyp_JU4HVr4=atUrK;BEgq;ZK zIk#52F}P%KVvC$fNN?b(TLK;A26v@~*R%CK{0z5QpMoP1u(YXCZQR&F>^nXsz^$c4 zUXN4PqXN1h!hFqv%%s6jJ^An+YGG+HN^@4Y>B+7t-0XhP^I9A$!3Mn0>5HXMRfQ)* z;hAT2leLenn8|qZtG!w4GNgC4Y5HL*jr+j%1-9bORJ|RXN72IrRrKayIVmo(-_MBC zpI#7EU;q?u)Y4CCXM@@1PugJ)iQsbzy;IK;9{*Ux@X&P&XrPN&AGG?NPi=f-6AIcg zV2UI^YK9**31Qt;j0~t8o-wg0rDTdch&|vg#=SE}+O({=cCGB{{sPP(UQNrJj6l3| z`4j%s%^S(gej*oiE~lFBq!)W}a_ko$B~ctl-`!-{E5b9Q1LeK_wk&|TbcmTWm>KNg zHVcnMH?S33d&cNrHH(>^H#Bih;+zhuQq`lY>XTydpEvxUJDVSd5o=Bf#n_367IdGE z?`u*J(WA=qfI1}ro9TTxuM6@{G~*v9RE~ddQ$xL&ZYB6;M$J!tFAlT{V-zCvejKIY zb*4a0^=xIwQCcZxXGq`d2eQG@=P+Q)6d>G$(9Mak#L!@phf~-giQ1^tg%l;wdMmHl zvKvS6RivrZJ^g$RtnmIUez;#JJtj8Ie~*)THZD)$n)V#!bSCzV^UD`Vb+QuR>1tyA z)xuhCcpT|Z4{`XP{B1J5PAx37n9@&SE1KZ7HD~vaqghE+t4fcdweJ*13HuUJV1Xl> zV?(nB1PQHxpipFH3-v!@^DM@&sdH#3YkL2#LrGzS=*Hzg8~4v*d_`P%kl$oZm@7U_ zC=zovW0!h!(6{(7nDwu0Lq_!A6oq9yOKvu(#Vo;gEHn0`kBy;r)9-uMvYmn42)b;= z*BjANZY>qHkd4h!$MV&u6|d`GU3&XdKB0WVSVg^a-ZnJrofscl0vG(kHDzQFGWgNa zYvtat?)nRu^OACXhp}%3m$L*+(q0m;Ktbw^-5(Za;0?^Fz|vrpsSOx?F8k>Y%5Ou~ z>r&GuYM?EUw|MUo?2VW3h2#6bbb$b|HW$Qhg|$4`^5|{acJ;dJd*SsUoc4ZNx|rZjYPW$X)}r8DX+%$QWV8s zniF+;l=~l2hU>jFny0*L?!TMvaobFN~j#Q9#@@BAnS+xW|#7W{I*HQpcS31q+J zPHha;Td2dMCZ-k}o&E(B%w=aS4`JJ-@FziA?&aJmK)WL}?evhlH>b{m?%=<_IHWt1 zH~q(p3+nqn7m=TSPWV*-29bmzdqtD)Axvi{^*d4u-v+67hhzk8{lMbBuh96UOPBB*>jC;-YY0M#J^WGJ-r#fN?Gg$YS7I@#tkJ`MSKpl z>zUJ3oMK*N)v#&aN_Q}9D5Vt)+I013xnUxu%4s5jaTOExOM;zYQ`V@O5o51BqEY2F z3HM->*vOsearO~Tmt#FVqh$2h3vaM@4^ABQ$V;l1j?68G%nMt;dx=rl#~T(C zX~@Ve!}d-O`&5hT%w{I|Jta>t@7-?qVAydiB+sDW8$3P@lWp=0{fZ8IpGxWl`Q#lU z-ZF10_Ywz@b~36>I;c(jlXX3QED1rLVGD%%qB$G?EeU+XD)*>*%E?Q{`GcU3w*9KY zmsbzxTNHFPN0oM6xOL5fOih-ey~j1pqU6WvE=$x-__Vg^mizRnDV?Hik}jPCwMYD9 zh&MEJ8AQlMz@DICehOr7Ia@f!Lk8W zy+3s!X?lO+j5MC48BDXsS_b~!$Sd=Ne7|nV3X5B|4o@G7ntgXdCmfYtMaXS4WYnK} zx?-x}z!_bpRT3Dp({xp*c1M$Q{P_tR%77U;?8$X~cz^`rKb`u0Ygd=LtO;G1IcsHI zFyEqRF(h?`C(CXL{~uw}<*B=W@M6hPZ-tCU=bO{(QnpCd_S?-n2l z%-8F8>=j(^V;d<4Mdr@}wZxbPjWHK4O&>m=jF*xGA4s>m$5)FD?f1nK+kwV43(c~g zw<E{VnDfs6#`x}lU?3Oiw{YUTjRHFdZw z$?LHOj?bM|kEmoL10gB=xV5T+4)=c$M-f!15Pi%9EFx#!aYb2xt&zRP;cPAQuuGOh z(Diy#;c@Bb`qHe9l2B{RR=9{ulcS}@4);Ztqr+Ja4vmqmcHEb6S8%ScIhtk3WhFB? zUw02bp`(2<@*u+jBQOP8I@nA<+w5O+$Ft zn2OEBDU>d$TVc=L&zvDupein60S@<(pJ|>XK47HBya)1fnJN14E<4AEdhxa^!S{&K>DF0M~rp8+^T%XNMvIBK-hQh7G;kb9_;UJ zHyt)sFy!L@F0;6a|4?jMc?wHFLVypZSGo10dc000J$pPWU)yu=TyTxM&r`^^b8(7w z$xYZtuXLQLuGNt~Nc!)OUfJ+ZZ-YT{O=rdusgc^~LANkNfM{Cc7LwITrID9d7bXWu zC34oOTtpORRp0D3H0o`#SaWy$+v#rSWE;#4A9$K*q$5QM%JFn{ODc`cW_iy<^6L7~ z`;7N?O-jU_h>Nq%m6(9th}PD1+sb$op8lqHb`}h|0)j0d3^fUdm!g zY@-$?pI$I1q$X)y7=e(&Bf%1_!n;2o)A~qE`xPqr|0jK{T8oKky*figHQy0ax>utA z(~5ifzr|Mbe>>p&9EVEAAbw5t)BK9RoCk~W?an-|er1h2*{%+QMet4k7scS&VVmZ# z3ZM9wzuTTwV?v|tjd;!RZV7YOl?zDm!;3DywwnO!;9X4>f=OpGvNM@W<7&%G_giNy zv0IYCw+}hfvzEUqZc8(x)10&I)$9rm_E+&R2f-?IVsb4fh6AkiYh;0^me4R1I(J)4 z^sg_k*%$S3S{Lo>`&rvToQBak9aXCRTkQvqhwUc`?$l2n)U@AH5M8zbgZLN6_L+S= zaIrJp1xLZ?29`jwUU=!(MVpIN#@xn!OhNh}DjeXqd#)`4;iNih9M6=>i5YKolm6zG z@PHkts7H4A8ar1%NZBC(anpD3yF@!p4kwS%k|3_?$JZ<*p3%3Y#IOMv^m@aP?YCVB zBoI~SPKq%8fSv15?R!BQ*AeCM!tE5gItcbVLpEt}maJ995#du~qy%wcD3CZS{(e47 zSQS_6B6mQT*K>Ur@WohhGg7Irw4lUNr^Xh_M-s^^#(M6uW2M%6# z$+}N4y*X(Cym@LbHnp5dB?m_G%8iucn0vNMn>5pDw|$`moLrdZHO8qMFzY$nuvwFu zwG1;1?=@pEtw+SuS{1AZ&sHxhwPl30k;Oi`_73vAKmx-5Ks{19%$TU|6|e4+sqUJW zvses5-}9F6dOn*uKM(!dH)f1|-tQU~@Q-iteb_(tFB3jSOiWBlN_KvZrd05&wSPuu zc)aM=r(nM?7S3KSpJLL4rl-4Ky&5&(Ym=XB&TVv?fKo#^2C&}|nMvE@s%9aAL!i0s z7&ndJ|0U@hwv)AiPw3l3e>}3)%hE*K*_5bfeRp#$**VPOe6qC?*$*_$WkpgTYLeI~ ztVdVt&7cLk{)-aM@>-gg5f0m6%0!$|-aXD}_s{a&}8Ww^B6He&_iUS5i_N@;HvfT;PHqnJr$M6PGk`rJWtm|Rs=iJ*$D?% zFBdM~l0ui%d4-GNE1Fa16^{XO$~SwnQo+}Z*s0+Cl#!&fYEf1wNI-18Sjxj*SN*_G ze^y!h!OsYZ8l#T-;HIo41uRc+Psr`zF402h)kye;miZZbAqAEd zLB(RtH?9Wchz;e>H;XAaEo_}yDo@mR^yaje z`SI4pyz>_`wsu!zc~4L`>(?mpQ^ykAYYC$hqv$GOsZlzVVdhcNmU6lLSxr(-gqG5h z=jw;@>K5^?Ff&l@eX}WW^!uPGth5v(w{+Q{!#<}z+yUwM$asq&{;cfHk#Ju#_mDI?1E-lTYY!^C+@L09?RjNV-IXmz?fyd`%g zrMJmvXs=Urda;oqryb)0JCq&kwg5$TQygW5w=>2jq=M?jNr8b@Q6@@BQMh`jzT678 zrjFm6t&L)FUG~EJQ8IFlt8fJYYgM9De%1ffDG7q^@*gO7e7mo6Mm`MM4`v-njieAC z2TUjxOqkiNqZ^HlZ+{kQio8ICyPb`K!-?5HUBBP|NWU@GW!&3>jeUv%H!zV;U3g@4 z?6Nn&!^c0fN%GLYcu~J&ZgOH75=*j@tEdrFerM$QZqe&&qo)UqBaLE1j=~h;oEi~9 zrLDv(n+HxDs*+tgx#-X)j66(jy@@mv;U`_}5Rf9vYmCi#!qzjh)DxHnNla!zmz>@i z2f0#=+_%{yyB~F@w-UJ3u_i9jPuyI}A5Xv*7|o|BD|CtAS_&?~Qs2QV_dQti{kzvf z=PiuRd3g0(4RE!)CC2JvwI@G<5QX2M5$ENl*$FfM(3bcrnkk9va`+BMI*Z0e8zg~h zG|}B)2%+Nf(xQUL_tPyl|7`uTL{8ITWM%vBER;01&n2 zXV97|q#hvKWUbmKK^9&JbJ`-LiILf@1qaO@X!WEFNn*ns1o__%B3_p5ORstIZpsP;cCAn zr6!*wCr)8uglSddUyPPy1hsVO9wqC?M1%ZL_g?;2XdmUmB*99}o|1ekrhk$!$?sJv zm}$D16WiJx_=qNcJ^upJ>OgX=ODT?wYZ62djJrtI9`K0A*W5(`ckd$kP&h&>V_Q8zAKFfze;AJoqkxP7eWfh|9EFlJ^wW$Z-)U`= zEsSM_HSP3zAq44Gs1shq@Hg>hp#1?6J!FGRr?6?Iu*%e7ryGlVZ%3z4oE~$D?dH)% zq8o+xe;p%kh$P)v8!O97FAU{+;~p=a=3|S{Fia5AB6oqGYyoP{JA0Wd-TgX=HwXHMyW+m6(pLX zY0eh#!K$?mR67sYV*Zd!`gK;OUD2q>y*^x%hV!(Y{@2W{X)lM;2vuB6ivHE^5t9hR zkPl-k;s{@m?10Z+!N%d2$esKsZc5NrO2y=W;@f4WMRv>Lk|gR0Bq_9x1xbw|9#3{( zwLpv66(D4BMwMatD!MR1spz6xA`q^|-nBJj;$TndX)2aEh1BR39Efu&fh~ivJmdG4 zmn3oG(cHNxRwSL#2CV>Qf`pYtj?YHVvw=Rvy-&NwZ{TR}=`nNz20MQo*Dc2>ig<=y zr4W3E6D~;B3$6+H1_D$k zBUYgZ3W;v@6~hR4+42kz!u{egPYQf~&r;^Fau@n}VLsvd`Rn}qaZ8|(LU=QtV8lvf zlxWHyB)d{s#MGt=&;#0wm6gHjXh4fCe2GH08V)>bm)O|DAmcyl>yQZ6Muy$3UQjDe zVW>$o{iY_)$pdZcZzOna+Q7huN3*=^Rpg&&12v%m6M1CI`1}6bxaWV>G#MmiiG5s!vHh=>-;V!BGTBX|s4hD! z;JCJz4^qALLBEqgyRyezBMv6)S9$4$(n71%!be&-iH$P!yaJr%5m*jB6rgPjrWC6r z9KCSSdCy|3e(S&?n|q7W!^o4p8Emz%HH;4dV!G5sPg1-~gk?t@N{8ypydPreokEL* zB(gBZHBY3VRYoONv{0`cQ5(_$B#1%YBow~VNwmh;+7)866y)&spUsa$4816>Vy^ib z3DXCVAK{vp#UYU6R6XPPM$%y9`b+8j5S}eqej#1e5*pc1Mvq-2E@y{UH)8vA zW5SE(JU9-gmPin`q8?}Z?pF=`Y9?=Q)I{tgQ|dekkG4I<_2-NI8ibDO5|#(cn`|_D z74Y#>51UsVRqHLlc3aTar9P!9j9(HJXGku-Rt}_t#UlfCG8DAYhv*CZ#_QIf>mF7i zyn0n>aGs~Q)4#4u17?+0sRl<`gmpOqc`3n(Z55xcW{XAB0t_0)t(}N^a>yFxS{(mC zc$O_A2Tn-5Y}mr67?oKk<Ul*g7lbDtNw5XjKr@Cb%YFSZfF75&fGgH=wG9b<(q7 znU<-V@k^Lov-O$I#cpizBZ?ff644^8tAc_saW*7L#zQ^=jPcz{qM`bweB?l(dbp+oufSLr z-rpL0eu(jqiLmcSOngO^PDW#WOVXho1S`f()x?P_l51}Tu|Kvy)K9GTEaUD6LVj57 z57<=}!<8l+B4cmU*C90p$dfzx#m2qu+dX#ZTI$W&E>Sy<+J;WtZP=+Lhfodq)fMYo z35&kuL?1dGhZf*8ml!Yg;NR==oTi4L?(Si`2ChY(xN`NLxdv(RE6YidqP32I56-pi z7qka_i0M7#;i%??ZC@GoaSe_kWKIFc*^YhH-HB1+7|jzGE9EpXmA`Q-=}|-^VH%j$ zxINi9mMA({!vCUxI^g7jkm3P|VWztj*BF}V_zOmuDT4<#{>AZTh0eAw?YS6U@rR*h=M{%~DpZM>;ClcWdT|rdIxC^SFxMLKiY20i7J2V}^ZI zWZ5yPlDd}ZcuQp??(4q2Bx){S#^yRz&8=)G}Qt5to)tC0u{ktv?Y+W9`y)^ z24q|F!MJ3>rbxFtS>NBhXPpc~zl~t#sj=1Kl|1#7)Y zphV{~##4$?G8`mSdtLMmi_u*QRX0j(;k$Y% zKL`wU16Hs`o<5}QG$~q^>JSYLh>0sh`R0gSoB-aVF}WX+Z%|<}ti`v8*qe)efe%s( zUTlXvs}Hx;P^lBr3DK;e8Trb8{SLQe!a|JgL^_^8;6hipGS)HGw`Uc?tCn8HYogPb z{FSzu{#+@ZVK%Q!?=Ro1bl_+xmP{%tR(MMR1N<-Pbp$B!fRj7z>Y$Wq3QhhBpOhqO zhR8nxEGEEdx8?q(;A0QRGNoCEW$n%bu&5z)sxOtTt6|T$>Gl1@8WSaYwfRJSlNHJt z%(h1Ccp^_Gx0C&;Rwq`=w+)lI+;koP!u20DZ6yR1M ziNA53Cj)JJx+{flL^=U^7*SU)DC3J*js=`^j)1gx1wajWVG^*0L>pLc(BsqmpxG0d zX}lhnFlX{_h#!d{IzKrGVcuXV&)PP2D&+>afizL zm#aVl7Mp`49ZG_Z*JUXI*{-f;*qM=NiPzzbxI=A;j5nV|9gfinHLw682V^q%l+hf~ z7089vfX^zhL+)S-TU1E(i8eNeV#p+~$uvCyl1Kxewp7_tG2+Dqyr|i#Y%IC>QRprZ z=#!I`h(yztqhD$QQ%OXv&Ulo-!06rZsYPqJ=L}DnMOJ@hK{i;nw7A>FrNDX{M#kuF7TFvy93$PQ^0M?r_=i}sF(5l_X zS2yL#DJCK-8T)-pis<8uyU~*PCEjkY7k`sTn!*b;hEuGKq1tC4s^4A5P-gMTZ`jS< z--o%gw;+NWTuIYEr*`}_Y~1MgJ(#%WEECFsJsj3OH-uFd8-&UnZbxZe^z5K$@h>C< z+|3~boM}~2##SUt0RLtB;!u`nzN62Ko@ax5({mvk-{W~UPo=cv{+pl+&pym;Y)&bL z8HBLmcDsS;yyjSe*O*SNppLJ`8D9h$q#{Cv=yH^q zhB@|rin0I{hk2}9OHdGC`O9t;MjwS%Z_D4Joq1kpJr-g!UZDRJA_p_xXWft7okGo~ z?bZ+0$K;MS77Wg?l4t!fG1)TEejaWxP`(Nd;|=(x-I6=sQ% z^QFSj+m18J?WM{lB}q;?U;;6+C54LKU}`nS^{p=)p_$q`DobR`CS|W{2wxYF$hm+p zj`G|_Gqvm&(HutT8 zXhfL2w_85-ozv1G8i_&ATTz4Y<#`V7t;@5TfR4yc(6#i<9z z#`9B?rJR_&ozL5C1BDdn^25=>xS1geLc9M- z$y@hyqOlT?S$|4metOL>Xqps|@o<5%ZXHCS?c+t!b0P*oDRwtq76uI=Z&p=AHNCEI zA2Zh6HN4BE5?J&Wl-&+CdqQ#>i|jGftXwX*Xh>VLZ?qQg*#vVM7w~Bc74#ihn>AGK zOWMEvP7Pt^##Pbmp)#1nFp*UI2ymz0%D1rU}s) z8K2e%nCzib+$-Z91fE+Hw~h(%{is;0kS_(3;c=RUaZ(jY0wg4&UHa!kXGjh>GR$YM zSXVjcUS~yyLSYy$?)DRCGal)Uwc6JxWWUZkzoc%VGvl&9v2r}h-o6eGr@dW2x4)ha z4~Meso?V`}ZCo3PotKz3Ugj)($N=lMTF&+jKtMlHMhmirUq>(QBK?vK z+2qvPrnmH@%KGmbg>Re zF9S0UE`WSre?f!g^CwPA^(81BB7C#ZoHHG#t{1Ud;-_D>z7v-%Tj(qLq|Da=yhgq8 zQtNzi#9ukuWqGw9fWP5ad{wOfn#K<4x>{q}K2Rno+CXdi*6Y4}dC-3CfP?mnL>@;7 z;Ya{Q6tpoz3Nv|2t&%hpChR&8|Fa(^I$TzZ4TsOW<2PSxKP zjOX;YyAzA!ccUC%xEkV%yN~{{3dq&BfpXf)q{JqBs61qGwG$nd# zZEj{tu~$z|y~^#pY`e}Vn#4- zi;)WM12DS`_hb>{DJ9A?%QZVVM&BhlnPUxbrzbOL)pB$fbU3x0F9af}lOvC;u#+uy zGmUYs)alxo0{96ZjS@@m|2WBg$FXRIX9vyH#Uu)1D6!<&cnKcuxW7D4I*XZND@8RT z)F&L*%H;(CU8t=zkQG+*-qQu9BfgEIpY>a_%5+jYdTdeLH3Ou?yAW`%^TYD($LQg|lWB#z+ds8+SfGsmZqOk6;iK*e|GQ~Mrb zXlxE4dEb%tV4e6S6AQf=g1!LcPDC9@n`!t+It?ffBsQzvX8rU1&yPQDGR4FJ#$y?^ zQyeN+DatBFc<6YI;p52~A4KpC_u@CM_lO_3_Fmlq~A^SRy+09SNJ!!u`od z8j993bIgl}I*ms=vS5+0)hjwvXIWiH(1Mz`XDZo-Mn~?n@D5qrp?jD_d~XR|beLfa zLWT-^d0mxO2M8%O?^Z%^ehIO*HM@YsxCLw$6tXn_vO77$ z;TI@caUtazki#G1?OwKWp$bS!fmvA9DH6I^!j!zGzsa7U4+@JF2FA5?z$Sl7)6G#} zV5ll=kQ5jjN(!pffW>vIKUNf>z3&s*K1F&HD|1C61z3uy^+%g!gSNBL+b9^?AkPW* zI&Angq)>>qbtK&dA=}F-*Lv%^50UJ1t9O&p?mmr7da823lEL@j>OI+2pY(Cno9Nir zXVr%zvJFOBQj$~ba1=_G$yJVo*hb7qbq_pg_{)fa$DR2pM&P5VDmCGCuS#)Q?{q>6 z%``Uag1rMkZWJsWJRY8otY-TXjR#1(i}rDWF0dKAmzPmTycBYfaHOTNSq9@XRN64= z?5X6yR9TyjUPMBB1y1rxgh$eazn_j4$JzBe*a%$}vdq$E*x&v3?|%$6*9G2-y4;y1 zHC4OzIyQiC2^)qDfJ2fE|8MrVBHm{(t;9dip{evYdtV>|L#1cDa2@i~C5Dio@rp6s z24=R}l9L~KrYu$lr&@#~eN5LuV*mP;rBKI%1#7G}Jx7?K%sNMtHCt+%1?i$R(Q{dD z7L$BQtjF}PGsCqUw+OfMLte&(Q3Gp=)Prn-Xhp3IPWcFjdlbi|5eF~v;ze~L@QPNi z>OcJ*MAZAkkuLYITlASqroHjuqCPQj)reFvm7vs(JRN8$19mY3dLcurl3R9Vo36MV=uQqXW%2C2_AmLh)y~sq z1*}yoJZ|>?1{s_iNqWr%8MJQ0iuj=*iV957bQ=N$E-@CkMf=wuW(xJ*V{^?%(aEzZ zNve7{&CT!#R0>NwQs|h4zV_a@dc$|ymBiCCPlT5pzCGT0Dti~O?^pmB;!B&ml zdI_=$)AZ}F)A#2X(*Jj!?xgAwfq^5d5w4B$N)<~#c8N0!b z%=WFwt(uw2oaI#bRZ!s(2`*xeKtCBjo@@LVk?!9oyMdy6u276GW z)k)?9mAC&F9vwv+r4b59PN4T7*xRN>;Q8`Nca*e=+oKZ2tQ1WeVK0gCDiv7zexet) zuhEVA%)BJ&Ctl-Gn%A%4+3O@iyMtHk`s*=B?xB+}VU?>!N?UF%y`qB~H16T|cjTg1 zFp^hryUzeN?!Nwa;JRWq;eoOG zeTn)ViTW+cC!jbGqJ&Fio^Z7b5{T>rUF3FPm9x7~Z#fU|(M5eSiA@?Iq5k7l~vHP z_{eAA3>1wxpK>DjLg0|L)(c0@38ep;#R_EYa-lW#ln;ZTph%%Bq;5s!&VXo+!^KqD z;hFg32+@&Pb;TWArbZ{0i*3f5lx)0NG&bA?&;2)wYdzLIBi)CZ^>dPk@V63-k1zgO zb$pq1kX`z9Bs5&oH)TKBYW@Z{38T#t{Krc)^P7KmZh^&^fkms!vGQ%(c3!R-jH(!9b_Z`DKX&BFdE#?m zn$baHlA#Kz{dEAqydo?CG2uZ^iS6m#s)~pA4{`Qm7FVt>&(XSw?(*aN>938cO-R8b-nyXX)i=&-^C;JqK+08yvYqVpZTuF{^gL&N>%0y>d{K64>w>>i zjZt_XsdAP2y2{=QwxMm{tB~|`blTbpi*1y1e?V&RMwvI3FA%@P@=6r~LvTKK%-=p( zte>s`?7J~8FM%ING5{??Bx(%g;wIe8kycCfp89tu#6eGoW$q1o`e1Pf1QY01u3`Mx z@KqN4s>S+hgtAdGOZ5u&cYeR~Udn68mn?(`m_RI4eKBoVc&HVMbT|GVJ|8!Mz}c@m zkWL;q;7#W(2)3K&g{>iE1$*G*j-1zc+M8G05CQx9N$~fWZSlromP1Kiu5zq)eUE7k z=D{qAhWGt(bnC@(s*3OMzl5%nn8U?mS@AnmYpU@L`5I>yp&cj&od>CbyXP-wZm+Mq zkyw`j-n>+^vgN|LN=_~m7GS2Zw8<+cP=FYRA=cDsmV%ZAP1v%Ymil=$&}RF*yYD2i|blY|=lYgvHwOE_hKfbj0Ha0zYUSCs~HRn9>4e(Q5 zVH3eOk>K!Fqp98z(JPH}J%6fUVM+Q`%5pHAd_QPhqhcK^|HIe!F3~+~G5|6bGAVz1y4T9Zgy2 z#=1>Fy-|J~1+>>KiCL>Y0cVJWI$~31>deTgJGK6ZOd1Z5%mX!6;We>%RqrcpeulPa z*p!`sR#(h>anfLzZr{OJyEQ+rVMO@_#f$N~jk7c_K~v;JE=)q#Z?wdX1Q(>JB)72eds<*~ExRA3w*PbG zu;OX}Z zhoX$~TW)Sh-YI*_(yYkOcSsRAtZtu`hAgZm|y!s`KX#Wz6p6hpSsOhS>I}9OLrSrc})=6IX@3x3IBGe-kIs1e1qLa!{Qmv%!K%W zT+#?@#cBz(Y^i{^&aNrwUEGdiXm|z`HTzC|jdFbJa(AbK?bS>i>Upu}UpRCm_0t$tgZ}<^Pom0w?G-LPSl>;3Hw;3v2<*}pG{IK&hq-U*jk(& z#^tfZLQULs>a8LZ)B@CliCh(18O&R2#UniqBS9N(An;?hSWs@HE!)r$MSdC0PKtOd zaoFT4=VF9>8dPksubHBC>I?J*S=Dq4 zM@s3EHubN*afMTbXEPhe^uS8E4Y_gb!14M6SN)4Q&TQWn0JP;K#{Vr2S=Z*38n35g z0@0f4aSSk!72c_or3?BDuxEU)|J1%;{(eIYN^fBUM37%S$;zhFZ&vsR?_U`Y$Kq2^r;0h>l#plR{;;v zD@;tVY`>oqY~-5IuyvNR@L8hKt->FL_KUuS-e<0rw=Uj50zIvJXP4rqCvp418*|(` zeSnS9(ADmFSOa^!NnNG0K6-4ghN)2PB7(w!=uwbLRP4Wj@4uDD_FgOg47n})ofUs;>bDuHvO*zM-6_cxf{u4>PmHet+E#b{uU8n^LiZR?FMm~(An96X6R=UPVwGxlC>T`9aZ-p#^3azZm0Z)S&ztDc}#80 zU|F6aGJK7iK5o@BDQt*!CTiN$(5NY@PBu8jUBQsDJOQzt)<4f4Mu%Oy16ns@RW{;B zq3o>5y}3$YtV8JsUwv}1G7oNQZVW>X`ll%MVX8l}8}Nr|ucUo!vPrQtVks62hWsWI zSZjJpoPjccACI!#Xp4OYQNrqE_ufg-;_mdDl+JPLmU`zgMhQH9wmdVdGJWBAqpAIq zC+}fQ>7#7xCYIe$D&y(BE2EItYGF2xQ`W1cnB`s}C}u^|E{8tiwH*q==(#%`ik>Cp zvsnyw8l#N5TpkA9|5`=2BF^xQoV0Dx3NKrO%7J({bK?(J#TRCFD_8YDA;8-;xB_lx z1YR8U*7sm6+mJ?QLZ-=C55DLRv}o??^bpLBk^4RAM^cg3!q@PNGIm|#wWh}0ZRu7D zvCs?#$Bh+Q{f>6y-7LJ$HD4{l-v@4eG7())Z+FI6P~?rSo=zS~@8U$U28*ruOm{m; zchhGo{6NZ76~2-!KRUdv6KWg94$UHehD5#?HibOwq;&Ye(CUvw#KJ0W)MjCKpp6y> zbZ3lzx~EwQkmu&)GhPlx?ci3;-XnUi5ofqesSMu3)I1+z_xb#wmtVGn8e(%;RO`=wodb$OE5 zyzFfM{_qaHJWIXs9DpfPlOL1z4i0{2RIllMl$A7YcKKemcdlfM^v?(>EgHl6$;MRwaHz(5eegnlV8PZ8&7e& zX(y2Fv3L9$;(2`84t<-00ljmHsIHE*4Exrix`KGwvITiU+_ID3n5&ygx6cSh(aV}2 z{L8&jhjs+c|Me87-g-)4#EGpjlj9kCAVVQa!u9}uOk_%Bhvj+-j1aL4xh7Xq%UBqD zt!T%{r~^&Y-eBK^P=ItZtPM=bnP%k>v<7us4Aj{LX}n4!S*#p71szN2?tT7%t+GkL zImTyzImBT*u9{kF&|+*#%5}{De$M#BRzRsg$O6TLQ^9m#kC(Cm9HgluVo@C%(a$}u zJq9pyaN4Awoj7e52613v6=hf$9>vWf0(}OoORb^!PR$99O}lRxxT@a;fhN41Qe%m| zU(Q>Ji4JOULX<579Cl#G)2tkf6RCdg34AYp2Lv`_aEng!mpaxUmWoo06+Q_9Crd?* z(Ub4kN!7sN%VA12NO>$fy|SR1Y-OE5NvNB1$^dq%_%&V5VGy&wqK2hYd1`c8H#V?- zn95Ad!*Y^tWaus{2z#8s{hLZFP5*cA+YR1`Md#(l-%q2nxMbb_CxW&etK<&$d2r1k z3!pi7>PpBMbL&GG^&q5={!%i?zX-^j4(hyAu7!w$Mn*;tp}Yi(OPu%>b> zY(zO-mGiEy2a3{$DT1>#gbR{Aud!zP)+CZ-i1f8%8)qC6^L<{VPzO2KYS;|f3Qn%= za{#{aikErL&{7`eWi@%$j2>|W9eAQf(Og!hrniSK`w~#gby*F0O;SbGfzzsKeS5f7 zX6bv}x*qeDUJ_t*V9WTtL>l^u6D(=~wj~`|^ zB|a8aw9!aRvhWl;(`U4WN{=538#`bHDQQn!mp&LUlYN9lkHN$#!I=xIvdI0tjskYmL6(OVEC$W(n3EoBwc#=Y`K9HzWx%q=d8LqlN#!lG1|n;gnPx?m z{Q;A`I?n?I$SH0`h8E#;II>i{h{n?rLLot`-xTC1|Y zr+03Hj~OdSELBH><9i{esJSFCfm>uEZ#}@>CYp>>@ByA!xRLJQRo`i-tYyfclNv% zHaP~N!@5R!L7j61%QjO&8#|j`P$H-2>c$ELs3%44n|u4t+xm zA1WioyG@lW2Kr8GQ>!*-xxel%ZR@~D(T+}H z00<|BA*&z7?%s3vV*biR-J>8??GMkKjeKE#PvpFAhWMx(HCnZLiw%G>+mjPAWbBz!Q>8xS0CJ2oks_XR1_TH(1t45*X>pXtHX4$P-++0G5> zgeFT=6Ad2CgnE4Nk{Fu+(%`~0S2hq8qM`6?cnYEfZ7#n}Iu!q!XwBZ%L6!(&mS!aD zN@lD4WeH^`f)QH|@1OEt`1z25m=3gPNKeZ4Nj^Ar2tf2f3EcHYVyv2|C7i}OEl!i8 zQpNmy=m^SZ3sx81wvQWQuUpOKKiWv}7KX^7=}?-CNMk5kbsT!L_1qVyS6jB1%)(!2 zB(#sl(|2L-Mx6Hf>(Lu@e7|sNX3V6v#*O|-%6&jpYdEa#H0TIbdJ^*{ktd)xj$8(N z&C{h~peBPramXi$QBa3TRG99AL9<=Wx%Xw1ef}v*SexibfgK(4#8+v(PVe#BE%moL zH)yuenVa;fTW~-Ry&Ru}oGhV0J!RU~c1_Ttr+ zI3XIqvS-AYM*uE$Z?tNKN-%%;Lew(b&oxZV-gKv2R9fPK8Vr-!iT?#KA%~&VlO`A3 zpActSFVL4%=0ob9b8zbQpG3P_@JFj>?WpXb-pD!y5D*xr-3%CAa~r8;CNotX9DQqz z7uVm91PV4ml@irJn^D~BpISS-(IFRk5l%i8b}rc6RcyY^(3J934uK~qL9D+i?S2DM zTY7lW7-#wXZ*_c{^BzLmFvMR)rw(aOD(KT804Uld(Blcd&A%x{=Kv z07`oAGL66ly7T|cA<0*zQJ7V)h};3-@W~{94{p&6e4WJB34D(#65C0A4(13sak#O} z)HOL)Tp&Rl+Y{om>Xka2#|1him=sXukelwW=2K&5DW#c%FH!5&>A1sR(F7JG*3c=a zLwg*4>Q3QBcsCqR_B~Ozu!YwKL?fEHU@)~W6HGs2S+VY})@%q&8-_;?uFSj7ru^x= z`ZkaC0F(axu1mE`ei57HMy0X6>h819)Z?0J8c>$u&#gJ{#&O1g9Z-kP!JwYlYW|A! z^Sh1_9jJBH#NF_84xdxEOb{X}?`&^$pmBj|1a(8eDrNl=FljIHxy;S#wM#{^=;v^< z9Gr5kdnF$qXr}!X`(n18P?rY$Y^hG?fb+}|UpVEM^zd1@76d5wjYSE8a0D~GOuTD9 z(I0LAKV5mI7fGe@I@{n0a2U<~uEf&r;L5OMR6C9uEl6JY$d-cqSo2WXfT^}tk*oNZ z7`F~I>xp__^EN7KgfqysEj~)z3>1N|#F#RSdE!#h`ANm*0rQtz)GayA0JS!0hglLR zfrMPjBrKU#J`kK{;n|6a+iUIXlygK_2K30%VymscpQS*{$O|ak?Hr#80)$43wCzqQ zEjiaXcMWvnfYH{%ajST?j8g_=#!Xo|jX4@FGG3 zc(wrr6wK(Z>fcUEJ850ubstH>zT^<8ae2a!#fOh!mf_YA0MKknF}93F>n3VbTYI5& zIi4ZuvRD5!{H%NmBq*-ELqQ|FPK|v%Q&Lq4Rzf-*KbD%-J%Uv9zsLKk@k%FoNL42hM^BhyOXlqjBC)1 zRYiz;CNb>WRXcmVYgNV6FHh8F#rs?v!bUNGqY;dI8X-mC&A(`@j+TGhj7ZGIdv z8`JU=u@!jC&|q=AKs;2!p-TJKOm`r=Jmuk39DCSzMUJke9z86ywvrq87%=T8ljq*F z64N71&9c>9sGUWQ0&CMa*k}98;X26ItSlX+XO1_UJogj15SoeO8Ly|0uZDOvDUdLn zmQT^{=dxCu=M|q%(-lqN@sk|ECk6T4F;wW698sn}^q}l1G6qE;*8cveLjJqk^VeE> z$bZl%O+kHcdp?u}3;U7v8$a-F5%)S3ZNc+NU(pMp@cpS_c6j7hTCII(^Ek|N^s|FD zO=W5@D$u|86BJOZXQOh(Q7Q{9CMcJ;-El)~*QFBfJ*_Jcvi6+7TsM1*mPr*>ELxwf zS@hk(;XGmGv&%7wx?yA4;PZ8a@NLtBt9yUc6!o3nX~}Cx@2S2Vvx2K4q~O?L3{f&1%wv}9F>BV6D6>sh zohJmz;MbNGFSv{ou2lm+r0bI);5suFkcfp}GpJpF$w~7M*g_yQ_VGoPfv9sMOb*B3o-6oU^Uq z|6+)tfnRbheS5VC{JACzdiP9jbP$9ry>>5x`~i^WpK{5|Av0KwFW#xMmK)+;#bLD$ z)bG?MWabUgwDpjttriA7UH|G2E|kW@ysD-7wY;Vf}o?>Cx@sdkl)UOZeB2 z2c}Ed-48$gFuEV5Qb*AfiFDqR6sNlw&*Un(NIDDs`>V z!w$1Kbr*DHy4`TP0b5FLqc{phFvJQU_3+lQbA+;$>QYxpc4kGkB>wrEUWvi-^UJH* zdKjIN2&Ia}o3{t8-apC^p0R`e_u!Vgw-{xxje+8iATvO~u)TkOKAhw3&(PTt{rd|c8 z=j*qBkr+m!<6pURY;KzHi+t&-;CCq*fT~%m+j5BI3n$ApgA5VvXwwOLBqor^nK!PBn|){dtKV&&7>kef@U^G;-_X?>^Ii3CW3*V5pn=H%rOdfwx(s z6dpm=ZW69?B}rQP{SbE!hKv5->cZ zuc=)I7L`nTDUM<`apVMDo}~Lc418i0UT@)Dp-Xe!XGgS_tJi+y?l0f?DG+n6y}XRX z!B_I8;vMG2&R~`FAC4wDkMrOSjpt7($I07EG=ct-ab1bD<&yv$3%_{z3C)^}6>N$b zTl3YDGNRrPI&=>m*n5OTpLM@9UBLALF$Xb#87c}@$h4zjA5B<1NaHgq9j@Ob8jhLY zjI7yv!^_2!Xc}OXdX1v>31tS;EXdADWTJOq3KtI63j2WpHR^_997EFh!_<~cev&Ey zzl0v5hHzdK@N{{WcE18jEMXq#pjJY@nW?2H>P^|@c6%|J_Sbms!CO`u{m(DtV>l-s z@zEX{4Vo(4%3jQc()A}9C?`1X5|%uHfRJ==Gpxq28pMG}L;2O;&dzQs8>wO61nhio zpR2351(6oPfO;0F#%t*Vox#kCv%`q^1NGJijr8>L2E~e9A#L-NN}T{iA%QsX)Tc+r z6WkVWASJv0$Xm4MAJCjPea{)macKuaj?N>wMdxY)tpr*U$@kk5xZ_~o)&$n{*cSF zO~IsA|Gj2$|0-vo#Pw=B8RHO2SMaS*^nxARFnrqPdTV+oirQZ4c)UP5jSsoC=d^ zAu+7ih)q-}&8GC(4#YYQuu6Ri3^kKw+d@|#^niKsjt%ruEM8=t`fjrota<%9^Q^xH zR{Dw*K0a&qnqd**CvyqHJ?O87ePw@j$e;R9=8ao1TKlV$1lV}s8|e6B04w0v3t;8 zE}CU`qm0Ki*>uP_`g+-YNm@D$`~{ z^J+Qp!SRU+p*@=cEl?0if#Otw5f#LC9{RLl{M3WrCIFU{vE!8e-bkjDvBQ+ZI@#Dn z4FReI6hn~V0Ck-I>{S5!sYHWRIUN1u;b4MQZkR8G_*by#mldR0{wcD6#uVY=OCE+p zFDAT(%d)sE%M<0kl%MrjYBboygm`sAFDpf$lP!$jIKtj z*znQcD|RC%L$x#CEE+VkOTX`9DC9ThG3QIq?l*!3taus+qKF>ojs#Wk&^qj`VeX-= zE0X6E2LmD(sS5`Zci4+1Fbe@p2+5Bn7<&yQgc-yUj{7oF7ql`TBBLv!_KO=h*}KOR zy9fVuyDO~k6Lmo+Kov)SB4df<##}Lgj*17CMdnWj!H;6e3oj_U0n=qd?JxtelIJp- zZ__8zE5U+NWx_l(ppufO5|W2Q7P6dpu~9=amHCnEq4 zB=k{-9fWt@MUi#{2q?oXM*)HgHeI@k3PxPO<42ed3qsfc`4e$&054Dm1&eGd@zM|e37X|hSQTP!J@)2&AhYfaQ`U}8& z*zav82W&J7@CiNe0=(x8O%kcwz=zA!3VM0+(W|$lU z>!c5z!X(Q`Y!j?Bb!~?m17?by&t;@lC!$-GeZ+f??DgmN=#@%U)8gN}lOu=KDGD;| zHvwf@QKRIDvhy@5Q9oPtDX{)R1Oq2|WT)ubCs8&cVfr0>NH%_Ue25IlD4;YmMjzs} zSD!%ff%BmtCD|?@Y;HhZvL#z`z6Eh)dP3SY_%OndydHo?-*1*Ut4Tt5CB0B$I6L9h z{dOk?h+^LV*q~-LUAQ(#`EJ0b-?c7|_OVidN`fTN3ubSV?*ptH?A3pA-AGsxAT;5N z)O%@&d|?YHODZDj#A;)4hL(JG>EB6S7#%^IwbI5Tc8+T8hE>}^asIV1g-VT^4C54& zoU|9ELP+#t#@OQwdhqkSj9^|q*QrBzGi#p^h?>klG5g)wMGejKF1<@p1b0L7SIyZR!!5o z7AcLhX5_hPhIQ}@VlL84ejb^1U>UU8Yhag!p(XAcH8@WDjfS`HQtB<_IZ+h{As0I4 zX2Y#HPlya8je}gQV=)nbY*Z0)-uamPt)s(!gRpjn_~f)jqmX{XU<|8%?|R14yEYAu z17)Y*3*|&lB8yXB+XI~yEAwtDQT5+X z_4*Wyd`_Uvcdpa-u1r4 z>;M^HiBckE@kGvbhDvZ}c0u9I_FnJ&>re|M$3GoI(O+c_JQZ1=O6-|kDFR6r-~1-u z6zCdshD3$O14pB5BNCoRXF=T65p;v$f|mWsW!r_li z?b^%ug){hcv!ga@G|zZCGoNN14a;U5Oo|HKKEPS0bNUH$ROR#@@VlHYR@qb}L3Kxn zkHu;pEOSuAz93~5keClbhYUvr=E{tv)a3MTZtBP=-3?Kci%&p zw11Y?>Xd6iFdxFy-kD^8gwH9P21Ft=EMh^zVvw1aDGi$2WY*~Br;#>{I%PexUJU<`IH{s!dg&+I4YeSN5q}*ktE!QpH)Q1k>n&JZURgwS z5%l%}z+P2XI>DGnaP|l?81OkQ(3-zsmnWSP>ZUtLmq(baFf}7ok>2pI+aS^)MAjVM zjEPiWW2PsjrJz@Zh%=KJ(lch`;|;&pq$fW&_{uv!of4zIM42WfE+k)4oa+^d#+p?G zSf549&MrC`#+Xdo$6S6+kUy^kW}0*Kf+jpbAjYNKVFo2C=#XCiHg*Y0l=3P0tO_|t)4jQV8~`zY5yo;S(t#`jQ$W`^Y4Fh{ze>K*zFC3 z-gIcElnc3_iCDeO{fgrOmzVXDz~QsZ*==_%YH9wElsmeJ+u)A@qZ5feJuX?Dh;Xq` zDRTuB$MrheQb_8MlSbT*pc zWkKZs#duYxTmCFFq}-#!nurU*w}Rr^7xK9WVG1N)dJR{opl)KUy%2^etn-Up?VhlN z;|GlI^D)_YzZ%ZkMUf*!W)3>}Le{1;Mv!ov7J5rh>04qNSoKR*=z!oW{q|)+RA9!C zep8Vw*JWEW6SKmT;W0#SLf%r5PAhMNcc2%Ij1ftkYjyv4$P=67D95-*r!5XdE2JXAE2-^cF!Qm7*{aAyxHB9G(ihmmpoP;$-WbyuC?I z&+L|*1y}=hCsO|U3ds}4eO8jTln{StXf}UOzxT?RVX)I|L!zCwxEfj~LkvjUxWo5$ z#d@prS_eCOrA)v`)Aq0Ua;EdVF<30NSW@3n4BR@OdVy688uS%Q67Hz2qF`LX)cb(8 zkLi=$KjUU1>1~JtLXVr$J)Zsnmu3GqYWkF_{3l97BJqQ_sl$5V+c!R(#jDThH7)`Z-s~}{>w)02P#GjrVu8vMAzs7Gm4pRxIy)t*3uV>9;vy$Ky)i*_ zkj2mzcWpd%1mNyx%dyPUCDc7px-bjNaIJBiONMY`~H1qZ_+ z&rFp3t3FCpX{62~l8OK>-OpT1lTRArwC8yd%O*M7r5&ff{Sq|r#w(^Rw^m6=RDXm+1W4$?yXoiBN`A zQ?s@oi#W0qeg`E0CuvOo6&jM*Ott}24spnt-Xb{w#lo~YHXD#jJeHEgqC19!qK4Vm z-*tT~;&w%jomz*Ih*oFWN()Ds49V@CQ-OLXdAOUvS)kqsfyYmfTm3aiM=f~3wfL4- zXaK#U-{RK1srz%^ILo3LE?{I(-$NrIqFDT`Ol;>@4N6iZN(3J?$1QDd322cQF2h_` z?8ZlC?N+L~K`ulAp$$Qeq0iJ~{>?e~rp9`VAi6b5=7^z-MaDb?CT2VuW;F0z6b@#CZy7-`0s*swY#Fl;2C*-<#k7n@ zvmEd8j~M*uq9y^<%1FG*NDF67>m$)A$H%xvj`Ja~C=Y`|IUav9goQL3c``0qaUYnI z5i2a>mhlLa&GdZ5)^R%e(VNyYMtG^T2-%$2VBx^S+N-F8+zI&br~;!{ueg&fnyzy~ zn6@n|;$CZ>k+3r%?&`wKDFh9IeK}9WO3U2XUQq3P$b)xmJwe3}?{B6R!}42dLDO-s z=#3Th#N2vcM-9awS(8EG_7cYpDy(YZYA(tdPgBr_umOY*r`AAe+s?$R9n;cnrrkTx zt36EA0b1xPNg9p-G6A(i88Hf=sl$efd}xC02x`O4lw6R!&LFSGUrrV@47AI@R@Vm2 zg<=dFpuB%&$b+)`*AaeGtWtoR|+YYnM75OxXw=?GXn}_#kX? zDmB0{tm?z{9ear4QO9ra=1zf>lr2gIV}siZwwo^vaK=cTvwNndJ>^n6bW`nVxK%6n zn_SY0%Q;sA{GTtyTTvOm!R@fnjcA;1X`({u>XlO6o9e!{d@rDyfTBKZW}3H631%yx zhr3bW%{jNo#EfR9X3X%}bhB};M=2cu=54PbD%{*ZNr~fDNHt8L$q_F zhpr?U14lc}$s3-(#QR$SCTH+=^4a7kku#EHwX9e*V%9eixu)>+ z1iEla9>B7?qIYP$y?_KF)!38ICQpYeHj2DW=AH|v{jYm^^7M4t>bmOUv!;uKbWPV= zy@poPd=0N0<_CrqT;%Hqx(+e~BeyL0y)DgGlq_MXMXtns%uSRPzHFF-@oGvg0aKVZ zNT6`^SWi?_P5LRU?I+qTF)U%}I%q?2J&x`Fvu9h|{#SMfM~bEPe#g^Hd$^QgD1-e= z4deK3+2Xx@+z6R~*91)*yF1FoEl$<+^Nd`H)Y`vE;J63>jU?}n+Wg}E5qNa0LZbn@ zo2+VIZELeF3RbE~L?rAKCA2NMa%+4#;{b`xQ1@d2&Zu%@MEuS;T)0aLF}K>KCvu0s zOv!}?G6qka7i}vr&x3Pjc)~Exe?V|86HTzfjAungGomYz>M$Lp&)VhZNhPscHs{%y zNgy+hk{Mf4C_4`&*4%xehupJF)mh_WWq_Wex2bGT(-XZx;Kq=P6WaM*mp@u_R!eRI zDcYCj?LEwU4((@W&M5d=s_*|})EVEL<^BWjxjvL54{7|@D~~nHw0Cv>@ucbPDMSVs zCNXT5vdlvXcaCRxY6TR@s~Q-{8OhlLoQht6!GS?*BAxAFdZy$CT1#fC5e_pwsV3HF za-W@iMkeIRsaoYZZX=jPS2WQ9-##_GS}LVL&4`2|0zkeaJhT-%M}ZqYBsUa0J10F# zT}yfMZO?6$y8WEVxk%Z?e9m)Tt*=DJlemZ`0BUPL=bX_?@jxCJ4m&Kp!Tw)?&)IFn zx%$JdEjXv_QKzL=bL50XatYJ?b%B}Qv}Di8g#1n9qI`~0Zw}Da_9z={hSxIX0;YEk zW(hIxV4ZlrEp=<#$THpD@JrthpNZhhum`Ps+`t*b(SX_xPExvGj*YzR$N^=EK_tVPLjd=fkEbjJo=$>)foRuOcH?Z<_m!k?4-W`3tj!hS(|e zLHetI0zjClAgg6CE^UAUPg!2^gl4Xk+Q5bd4p39oUI`~Q6o`D+P#}yZOAN8b7yNm6 zN;k^JQ`HAvjufF)2>(W?v6R6pomfqt%Sye) zr`S=aqZgSJXNuhz!kfl8EXxF+hqW3|R`j9BSd(XEJzsj>I|siH$*s#+r-V%x;8O1_ zt)q55hoz8JPFL(`SYd=wRVxp7qOm&a64cJ<v;eT@KDOAEfz$ z-vgj`jDZi0p-Yw(q-1IBDSZKrgwy`QwFW?3z6De~YE9%3mem;E<4h zpPoP6Q1Azdx{Ci5{MT6J&A*rP9y^IAvO)K$wNoMu_g~40&n8b#i3gjpMN#*R$(C*YGS?7z528$UthJuRL9z){0yLi8l{s%=`#C%I zK>)h}mjbWI?^dE@zZ+ikmd>ErfqtT{3AeLzE=u`;#K>TqQGx6ZBTnE-w_=Zm^D_HHMfs&#+Q80j8}x>E8R6l$2) z*)^+BTGAjtt(P|W^-6#1p^5b#S)QUv3Kw~cgqBEYZ^mbzOk+`Z7c7TNk<^y#Q_6&x zo2stY1xxglZyk~EG}UpaQ?gRu=vo|9-vxYgKx(8-pSFp-hu2-vY8fT9s2g=-jD1`u z(5ge#YNZ+@DF9XqxC{_jKPBW%Eh`WzG)Z#&d=N!1D3W2PN>R`Stk$T$bsuU%{ZSxW z>D^EQ3?1C%ix)CsQA&zd3)C4Z9W0>vGF=l|iaOT;5FRmJ=QO?PH1w^5`K$F}n^2{6 zV_(d`)9KSETMu*Yfd=TL?{W>qZSpJ_d8>2#%^Kgj6-sLOI?v*i={Gu)Lp?MxGFMgB z>UQnuh@9Y^$C8vnRHs`{pYG_bV zKbl~_pdKyvbVVLl!0DrFL>}|@UqEG}>N zM{eu@+sHNaLB7hUta$Q8CXzeV_ejG`-;=GL=H%kLYx0uIyKv=9teDwGsi3CJm{Nj3 zJTSPLHoAm;D1`KzT+P^QTbI=qV||beyE{|eg?g7midal~+FafQASTkr_vbRDPtTwU zxKjLST!(~GBGp7u=df5Z{A&w>cltiE=?cQM={@MJ~YwKDEHI+ zk%DR?r+(RyQJbUu4F=wNTTeG++(_xtOZC-<>Bo>OcOCOPo_o{F+q+oZtyCezly!nq zu9s9XQnISEiWh;prm*DI0)q&ATM3RPfa*fXONMUu^ts8n4z560uD!(GfS)1N+?Al) zA%#UArg(w|$=p6h?bG;SqMlBH zX9ndSK=|$mOyC43eej_bYQn|GHxoz;qy#Glpb?%uN+ez<1kJx^FbVbGqq z!yVV#FQ)X@x|o#co|LQ_b)q+qnDfx09pg`JOd@ZmaTRFu9=XuG2xzN0_61u-vIyWb zBJba63`CHeB&O4yFS z&}T=0W=piM9!rw(IrHE)Mxlp^oL8cZM-4kRPpH{_XXb*ualWM_5n0BPpjAN3aUd+o z9@S09BFX99zk%|m)wFY*fb(~+E?&NQHC?5@U5bZEB__3G6JGgF#t5jLNbr{!fSSVN z$Rj=GrfR8%5aY{f_MChp(VP!Dj04iuT$E~UmhH8~ zsssk%yq2t-;;TQE!3`aB(Aam(Th_F1Xy37U-^l8(t>@Rza8LrHb+yDue6YM~aiDjo z5wPr+AKO^mLSMv4|LD4w4n2z1XzQN1SHdxSIDZi39nTk&2VN~F+6Q_LrQOee8&etI zI65xURJ&hm$>w#20XK3=Xu);N-We#nXQbxoUqZDyMx8W4#I*5%Rq9fEHGa{nyP?Rp zdqkB+)+=c$AReiN8TX%3&T^*diszpUHwep^Mr7Gqe&l&NJn9{jCfwT=AoX|J31fZC zzGM(1J&4?Rq1!&_-o4cw{?%Ikm`(8QlOk=#|I=|Cw~cas0}(rzQO@?~~N~qt4A8hBHpa+g=W7_;Mj2n^R-k21PMl&QIGNAlHGGbj2!GHb^>& z$a7s`Uz z?(WC}_G^T&!eG`y6aZ{FI0TrY?F+Ap*}IyDk8{#G54Bm=N_{iqm1Q`7cOa_}=T8IJ zi<~4HYXuH<2g$>F=zQ6E>Jmw1(x5F|;FBj$o*x}inb1K6al|8f2G+XfV+{Um6Ob~sxh!v1Fd=?{OHl9#-M z)I6?n3rs|&2@cLhMJND7v0$A?M7zO*SSKm0_+nY<)N!?XGoDnKA&9I~1mG2jP1(`) zuCy2k9X=x&zhh)B|5}Q=Xpo389M(%y!_r|Rnzbld^%3Kcg7(SQUV^(j5~0qMZZLKW z%1a*{Ezz+Y5c+``B<+JWVJUfG*^CKNE=qw_KmCs6+*Hid%8JxNw@G<-Tm=S@KnWos zKx_SoE*7;}Q6=M2txiBj+p@(?T((;I64z3lZ%{@ACcgm<`870@8qi=T?i*{c>N%Tr!LN+Dhp~I$Mv&d>k&M>?4raW@9M6|?FGQ#_u5Bi#H@quQ z8D0lTfe{qPoXFuYXT0`&F5|W3>5NzG8I4!mNsU*?!-rS97O!^mVfQslMb0?4gBCF& zP!6(in8=j8(M%aP7oACw)fEztSqyL5JiU?nOyLM%*7i2w;t^yU^ao_hA(pSZx9{r9 z;rb;-ewOq+Nz1MV`d&ONe|><$+TbGk(|`&!F$0AnhQid zBLpGNlTU<(NI4zK1dR%Bt)kSVdyJ4L!Dh$#Tg-E>Wk8lAQpJM!+XLf zgUB3M+lS+g$s@d|bQPdapnhMsMIhHG)g9Q_L=SM{Y_kXpwcWUzjIj|4Kpwy^JqRt( zV3E{H5kr2g=B$ZV2$$}RSh^7j641_V#oAOnoc%bLb0+9PnLH{^m@5`<~t$-r`Q%wHMEp@sXjW@9-V5x4yW2B zI@R2I=q-L@0H^#`y0LzXmkp;w@wS`xY zx)D!f?jAq})#D?RC0kMW_@x@R0W7V)i%X(}xr!l}2`g8Ui1}G7F7N36X+U>3*Qmb{vZ?47x3)>n;_5j_DGf)!{ujbLS7^#ThwVgVG*EVj% zfc)Xg7*X51aUY&pOKjVH*b*&bc41if=n5DyRHHCtwgy#vt-oc zExP)S;o`E*1*K-{vSfLctq&-3*^vt+vo}|JQW|#|m`@!DdkeM}6&6AqS0+5m@_TYm zOBwDl882p~{GLk@wOrJRxc)j~u@Z#PAHn3>&qO!AGX>Yy!s%eEB4-X2C2U_BWV6{D z9hvv39~#G81+6Mp=FiE0oxK13&y&-?oSc0BWb*HS|NY7PDSY_d>0eHNHh=v7^z`K9 z`>)>o_2%l;-}&j!-{fd_1ovqMYE+KL@P32qI>H zw%ISuA!V7EmC_He)W?X}LGadg3l1(}$HuRGyy%o5A#jR9ss0PgWsB%3cStl#MSWSG`T}9 z$qwffoiY=LNshCm3BGBw_Y5Vomh{zVsR#9nQRONo-a0Wsm=(?_B~ z0QgVrwUTkX&Zy8S8GklLpNO4=KhsPTsZt5hGc}yd9O`Ce5pa2=Eycj%d1dyhaRjO* zq~3_qADBfm={I=OqR4mx(D$IJ!Zh1ua|4*ZAE1tWE!83yxxU9sRqq(vzz2ic58x=e zFnwEp56J}Ab|f295M-1Z^-|WLZ0H`K{OhGC=HTA9;-<8S=o_0c=UM!2Gyv9i66#zB zqJ5O@^GV7&TS_?L!_7pz#Z{uUx3`Zs2N~K4iW}_CY)xoW>Pd5d<0!td#u|O>CF?{Z zx9}*mcx(o)7)S(}*xKBU%1>r&jsYvfyGGFh{aUBPq%30`xt1{|AG ztcc8MA(x^8(*-SP!YfY+!;&U<>QV67LG#W?a;n3d-Dzv5_*BbM)Qe&HjoPP^j7l@Q zy>zvfEJ?b~X*hKkt($#>@{$cuxcwTB zM65il^nf9~Jw|@NZ`RKZTNN3t;$u0EzH_YRj`vmVX_}`KnyF=X_3evLq=4P2tx&9+ zrYfqbZ=?jZQG1l&0${5dOH-%P446_La;&DT{bQFH&y)XfsvZ3KR-HJdx*?)Oh$YXuZm&doU9e=T ztMS~TVoX?Gl>vmUD*-`QJjJa7Z7$hAO!R~<@~XLDLMxUB1C~uTY%VG!jU{DaOisXB zXo|B|LIB?ZUg)KHkrVcTOF%-wv4yFj7;J6@6!l8A5)AN-f1wJ!gnpNc>UVbOn;=3g zoloe!;HH9=VgWfA>;prfXL+_JD=sT~$EM_32^?4NrC#+;xCAm&urYM<} z;*RAuH9?&QH$EYdn5Y1p856+@p&NoEOPViOlghoKDXfvNtOgLWW|-Nab0uN(3Z`YI z1QP4pE&lr4-!pR5^nUMhktuXxD}t^pPihk>qhT-fa)4;!Wx&t9-fHo=*Ao+-wtr5A0v zfx`}_AxM>KV@1DWfRiYNK9|(#8|wTR7ai7vdn3{~e_p3_Tk~*}e1XY563kn8LzLTL z1yrhTHCwWz_uE^V8|igoE6qTYCh)q+(-@aXL`0#vMta&}@>R{Hc+_>YTd0HlW1rQm zUdL_N$e@VwROVt_v7qPVcP#-&3&U2U$Ap{{qvB8Xgc$6o_U$naYtnfkNM`+7s!kk= zb0!+eUrFL6nAAOglEpKWjZ+>0Ai z;ZS*RQbO+4 znr|37<1y22RvViDvlrdZW|j}t%a|NZbG3TumfkQHmrt~p@i7~K-oZWS#_o;56neL@ zk-;6i1yGYVuiQm$Qmj_C1EFGjKw2fzg_{ItWnHj8sJERFK5U_NFG_NePjs@rb>3*X z+e8qfO@ZcdlARXXptUafi>ph`q)&2@BiqA!;{-Rfr1dH{NXAMZCZ9a^gmqwoGxk8;Sy0H6&W;_#8b6jddZ8rJ98Vmbo>aqDxbG z78GTTCita@i)ftDOokm7fOF|(>2%JNS<}n0Zmtlmuf1LP#tkg5y+jXz5c+mO#SQQ*a6eG zr}uo3V|G86#x$vuYfjQq6h*9{q}KXrxAzbRQrI6T#0r~cZ zXoppw+x2oz-myg7vl17;s}BXuB^UXZlxMLXA+uB(rx28?YsV*y^g>#OuBUsY2S8(g zDVjPkv?$p<7j@h?=ro49LJ31wT3284g{|yrLX{!(rcC3tVGJpzRV~|CD+vTN?9s(- z=l&29CnjUW9#`WbcxPZ9 zIy(eU=J?6Ad4-!hqy>^|-Zw@j7b{*>%!E1HodNMFX(Ya$dug5k4it=QW*=;=O-BI$ zP5Xg+vM{=r618)}R|S?rfW1A@Ie5l#FnR z2tyIQx-oA|2qeg@=k_}6tp7*{^L{lUcY@L}j8#a8b5qaBoeuKJY-Q&*p`6z1(4fOv;=&`(%Q*uEh&leeUX&AGf^SWrVszpjq z`myHsG-H_0NON2BP0Tms&vu)v5Mo~0H zsS~C*!jqx?S5u`WT#v80nlO!$$wFX=ZtozBkYJWj(%OXWRivoPLP%Zrsk>+5I48G= z#wI-!vzF}!ffQ1mwj4$AqK(m}9ZUc`rG3kye;}f$n^00z+ia~HY$_&}A0K1fjkc-B zu2iK;Rc;JeCE6WGmuc2PWfQfX6ixb*|UFLHXK!L{;w%rhsjp z-VIG+U|b{TYMeA;fYA&b|K*&OCC=S!-OstAUXu!jGYxM#!@28^;T)44rz3PJxnj6& zWkoA?hV>BVnJJqReF&{Y%I7+Q@Y->Po*|fo#)EJ{n4`#=;oYAT75{A*UJ^YM-Dt-C zILhtUs3(&0KqqOa!jidq>UsgXy#YXLL2X6=$e4={$}8UjKNWNGWn39ahaz7g)5N3! zQ4yXeS)F415VBCkt77>47DKv~G}1h3WI&VeVnd5!8GIk(g~STGW|KIag;Bkal)Ph< z>^W*vL1D`VBM zBugPHvZBe7=N2L}w{Y5l{E0#giYhT?>yf2tk9BQYy*!)ZuDI=+lC^I_FfUL}t^0ga zbx*Sa7#dV(;c$X`{fl-7z83IITbPs zWK6{zr}vPQmxxtSiWN6e@@7r!48WV!-jYb|8P4ilO>a|a=F1nPqUC~BiVn%^)r@&T zP4`=4Y;@_)V_Unk)r;*5ZxZs(P=pL5kRj)yOiYNUDHV!^`i++l4?G<)7;t-g5RB@p z27#ziAUtJxr8alzm(xyw+9`R(5t0vRLwT0fjOPu$V>1CEFOb?-#?UWWg`i;o`XH($ zk$hEW70sEbWw!P%r;j-a;lnj!A8;eY9kTZJ_F)TYTRbrn7E~KdfJ&2bo1WW~*MjD; z4=mAOD@9f>_#iaaN3Dn7l=iMAfPDL>(y4^M>k+YCiTYos7;w{Do}Q$vV0i$~OXTFs zW}V`I>^%0mj@K_=s`=Pd6lc))l2X>`NF( zf5pN8mbedOD0e&z=8^U&w7!nhFik=`mf3ID<%gZga>=1~SZrCoFDn($h9w;JP;B?0u~sMW7q^vySyFUJm(dd3KX z@VPxy5gtEq^Y9;QR|Nxze>*su)^5EF?xID)8k6ZuF7M*XaBV!+-dcZs@rvkgBQ=+P zJ7LIBI80VFi19ntv{@vqaiyeO-tAJ_YkSjUS6?udCz{eQ-u1Q?HF?kg<62L-;l*?E zDwlQ1nxa6R*M!z^q}D=SA=bQnZuAo`;gKfHu9?zHd*7?U`?4r|Y1bwi(lQQXFD+10`@y-iH~E!piy@~@A$J9SY3s6(^OW76%alGngC@je z>$dwzB{_O}G$q%3#WPxF2nc3T(ae2*lej2tLm;T_$q_jbCA27^Imws-7SkMNjaorK zXZu)ekL=z2U8JK#mt(x&IZ|Eo($&kxSrt%A{`%t8lzgLA8GS4xS5y60YZ~tOm;)QN zb$753#OU+>GoYjEZ~7d5i2?K56T+GauBfeLDqg)~#bCH$N0P#uJN92*sV{`&P|B26 zgd#IZ>WupQMri$=l9#Wp-o1Kpar5fsbMjaH2JQ;aQ*t93bFoL^u_=GLponeT88Tr) z^QMj>R34x?xy*TGt6ckKza$g64j)lewY7S|A>4=M`X%b;)>}M88M(ruuD0oehK49@ z%9KEjR}yRuS_4dk&Zp*kJd%Qj;#^oe4G|?4%hRRir_RH16FtSw z<}_;HyVtw719#(yR_GF6x($g~t*{!+(n6la0bE+nv0ER)oHaQ0VeX1mfaZf+w!>e| z?Hvu(eyvl|{h)&Rd6RXJLh*pdbhN*`(Gwo&KrUR}I=^J=P zj<$KktR3@(Q^`x?FZpFAJC2noL?#yfHJC>X#BVVec|s>@gm+hKs^u7srdpZ-X?kX* z`Dz)RP4J6%e9uB*cQiu&jTys@1@!@|rBj8gVC9Nq?oUfv^y^52lH}nAcvm6PZn0r8 zbGQnBkPRx1bmdvx?YT8~x0zs3vWcxeGv^rt^lo$34*PEN#?)ghbHDI+q%=Hp|vcAwX>%p$n`GQ)*91QSy79v4zhNxa9f5LD6!txinr~+mQc8{ng-H z3=)R@@%HN*DBKJF;++*8)ez5;AY?L`fJy1(A4+CUvs3aN)9;7$hN|RAMIM&ImPv)a zg=ZADD%N|`PoFCaO8&&-u|*JNmzi?~3QV;U=fBRPA}ZQ|y6@*mo) z2-QBaKR=HyU*41|G!@t?e&4)=ti2a#AIQ*T1*-9GI-mU2z55d4tpO z+UCIcnzUypAgt3^cc1?2~F;=vqfNFRp-!B>(oOCm_jv_Uunjyp#WN^G-%vl0K?;ac6O6 zd~9(mh8VCZl$3Ri;kAmF>P-%dFTfDe=G2In(;j1Ef90{UCcOz1c(!y`9rLFszx zc2%A7`QQi~$lmGTPq&m*H7HOVuACmyHs|Ru`Gwq3s^>TwAhwFV*LkJlZ zrH21`ww2CcGk`{Y*hnUhhOV4yTEpvme|SS(m??l?*@<}qg>liL_K|M#@xX9}6q3!M zBvhJHSBI~;S;~jt%sY=YIT+uzZ2P-lmZ*17diB2j=zL+GVb{#ooM9;{k#CV%)AwN8 zX@3QGya)rfH7f$+m#r%a>I~~MQRK#{)OIQDut)!y((3nV(%*Hf7U_sYHg&TZ^UIf7 z!Zev^QZF(RvC7EiN;~+8o?gD(dSi;RdJyFpkdF2vod_EJTQ`YxSR47Y_TDw@y-vNo zp*`rAh^^8(atr$XAlC2J*@B0a<>eY238mjx5pAqD_UR~c#?{T z{DGFKr&x4UngcGx5G-ap(V=+;g6qJitNxhrN@G83glI?C_Pa{*#wn@Th+-M~Esk9* zCnBk6W=dPK>VYx8H2LJo6SO@(`TXN!7o432P*JwMOi4PRUCr{LZ!Y(P|m2umo_)^SJ|U%cbEA z25^KgmZku?T4NaxAjyL{1?|EH^9I8#;XU>wfi9epjNdUbmr#1#pG#seluIU+$gk0> z@kPz*lFSQ2v|ndrL`~bahe&T`Q_M(L*D`_R$O{6SA;QVmdgqm;iqoJ0WE#dpGm$S~ z)&l+!;RYMA)MoLni-5-_zGhT{(b+d;Evql;Jk272V_UsFda*$PV|#UKhAC$qg+Hw3 z=ynRoxK%^%wkc`5`LYPA+H|h=6U_;&D=`5WdJ}9tCGT?rxXNdMdmIB~t|h3=_BgZe zVkyU9-M-Iz=-Jk*?WY|@GgDP7wy8N?u`|VO4Zv|5El~1B&Pue$#1)OAotkYhgkPZy ztm{L!1ymQgkC~-hF^Y`;)Loj^Nfv(ondd zt`PNLh%mcn2ij-WfbRM8eS&JIzDGqM)vD2jJX zGb&#-&ao7M;MjVc+g9YwoT{uICo}V)O&MRridG8^^0x=QWFP zE2|kn(~(B0)>`F&YU-%5k(AY{XsPyb(>8Lu3j;Skehk&AUDoW3q#{4Aw0WlxDS|)M zMZAjdgz&4wbaOIJ*z-s`X z!hK;g=dp~&Hm=wH_*R=A+v{R{dn=($H9xUZ^cj7|EVYpDwYfs!#Ld-OXYWAsxYzC@ zD8=NUc9lXzm)#a%AVufj40?KyJ=ILsr(%tB%-7TX7|(7rPzs`)%r(%yE=bwa@JQ+= z-LuY{BBd=m@BZg2UMTej=yu;8bq1n?{0%Kex|Hm5ZXE|gXQHl5b}arLFH?b!5Cf{WOiC=s zOjfkKW2u#_VrS_}4W2bml;+TbC_K7u0c^$_a*wCeV>~x|KslpmjIu4Xw(uKTNVJG3 zSqLn#0>ELbWxPWUFB(xt%e}U-KaaFM580h{KlN@JW69Y{QC~&Y2JUV4F~q2}iVoSR zl~D$vf%qv*fYTKL#u>Hz0aP|fY81_=fT~dxZSYkfz-kLi@<1?_6L2+(s1uxZIR|Fa z3xYd}rW;dd1viP#cqdA0oL{Vo=n6z_jUwxd@ajZyjiT#`_=+h8Zm!r6u#Ij-g@kpl zcs&fSc66wNJ6g6hvr!&lXG%aDEga@KC2#bBH*)b%B?C*5)s=-n-TLIj3p!>Dh=^ou zNH(YG=x*yHG5!$NkB&be##-FRSlc7jA-uM|v?e%D z4%ic%JASnmg-f(3H~~!LWz(&1$A~}C8vXXl(?-+2R7YhK#}R#OxD#3lowCpzzD;q^ zc&Cug9|lHqK!ptG2f&WHSDU-PGpC=ns-iU}zSVdM122<42WK3ic`hmphB-r1{3u5v zv0=n+UUunZpN*0ow~e)G&WD|XRx}MWN-|lp6&)>Ssq0HMGgn%?Y$BOS#mbc=Vm?0j z!_8Pbx_=rOZP{E~2T06KS9TiOD+fRtZI+;7XJeHg;-CRo3z}l~_c(VSX;`+4{;)$i zT2LMt&at~M8n4|fGCh_UV_g|FOFm1?nGt)&Dpx*BjIpkaqq&FFwhfw5scxf4hAc5K zSEDF~EitZjWfWhGi|}Sl zGY$4`1g=H*(#{-flo4gCz}E5cbb35yc5Lket+v(GeDZIE<%vj54$z#JBO9rN^u`9@ zTC{*oT3gTf9fSQF)4{EdR+Ewa<<0_gBPLta{|J<=?qy4oXnDEG7|-~M>ufN^PMI9( zXE;Fu)1;`+G_mQ5twgy#vjo+TBM-KwHJTbk8*W3K zf1aHF<>ci1CzF5w`|nTQPvOJwPXBWHv-#uqr>7?;-+%SyuQyk({?1Q-{ywi)clg)O zC*QMIe~%<`dis}t9of*9UcmEeBFYI)>2rvH97WCy_ip>TIr^LOEaMae0!o5n_aABF zZm0eQtMU)E?-$Yj#=BcS6&K0~;t2aP(#DJr4Vahpk14HUQXW4!%knP~tA-HnkyIZ8 z)qcf+YBs8y#@sev36`9(9X5<|#|<50OhubCXK-c1kWt$r4_0 z4U<6edt7xI_&Ao|@=CI7j#3fL8`$L1QHr&4xI>ds90HE=z$GJcDe5dGCHsf=N5?XJ z?Ej6^npV^h46*IPxQ5b#>)JDi;8W~F!An@|*DMivx{cv#gGM7VfjcbP#FEyfO;x|A z8Bck&wh>{ty_D_|L}qU;voK{d=Y$qTDQL1}(rNkGh}qT<_7>bWs~0>+r2r@^DKT4< zQdG2xUO}5(bXS#mrFuyuMHJbpat-BKgl>*A4&2!Y@unq_rKYs!zdcdKr|=}2l#G5|k&IDUk*80{isy9|3sK;x zEPwjLh;10JtZWHt;3_Ea*=i}#$gKnvX0_f5!Eq{GZABkl4?Nwbf9MASb>Ya+7xC+Vv7EYEh_kVDekFkaLE>2R^^xx!3WxlQ$r<* zlC0{i;zedh$J>c_hL}DjUGE5WOz#vc%V0^@f^t9eJ+W zf+lw?PsumZ3{JYv=}H&AElP3E)#7Anhjt)R7ERNL%WR<2QLNf^q-r^$V5GV^#D653 z{!4zJuoziX(cHBY_ukS?8{II|>Ff#(;H&+%4xt#;_Bd$S9);$66HszhPP{J*>dLBv(bdQ6b~d7f1A; zJrB=H_uT)8kdsISImU@p2;p@}bV@c0 zk+ruKe1p5nSvch^fT;J0ETcV_WbYOn^fUXszre3kfBmBkQ@GB0QnD(L0@5wDt{(yY z+;KG0x>~Zl;t5)IWjNr_WnJWQj#@>^QmB1X@{~zp`BSq9y(DvnKb>GA|aL2iB zKekhLtUET!(K?FH1r2fAITuIii^rfawiqOQv%w%~y5fQS=ud)0hE+0C|Tqs3Ai&!oRYMhvXt1b`(6;2XAXc&Z@_<#k(Hb3tF)U z8h>W%`_1q{?`WEqOiIH+^(ygp@#5-|{1r*CBa-Lo}heC>@C3TIsD?pZo<&LvM&gzn;?2<3*-MVsh+qCdB|Fchn@sOfMWdHj;rz zmP-y!Kqr<(m+;}T$qzuW%5Rx{Ef(Aq0#nNe2bOX#9_^zX^sASHq|9FBbe8!&t?dPE zA0c06bU_&Wah`Ii{w3Gfufda2=UNx=jso}=H2<|^eN&JqO&8?Yykpz8ZQHzK+q`4j zwr$(CZQI`Yzu4H9?WnGHUYPsjDS2ipX?4I zjA{w%7g#`)N#rKAyMyZyyPR-Y_e54reiS=Vc7&#ByOx?E4EQwD@gSrBS`#kGYt*k8 z5iT_dj3r>D_7p>MKSQ(U4gYG`4f?nOPf2Z>xA!mO6Yga3s0||{i!0!QGVyBXfYRfF zcsEUU&6smo&d6cv(Fm)~Q9=Q_egP+%I3!`*3R!%Mr?o`_Iy#_d1K9HwF2|Ly^3x5; z;A4^oXN1$IWJ{`73rb-zjSgZ8<2dUJ{`FgDQ(iV&rW5X(^!U-#RfItA$;dHH1r{@L zKKZetShhbo#RO({N!~v>#SVPc0#9E>VON>6jFR>Ca4hHhHiW``Op2U-&>Hk=*jPXc zte@P3!%r5|1X?rnC0FsuNGa$m76-Xpr=g^IGC6!IIvkoF#skfRah9$N^1;@=i~_m#P|WujQ}E{PH4NZP zCR`(tmJn3+SqInx_LCaeT0S;2WQ@98#tbnNI=BfS2q^#vSy|j*(hC_0ooS}n0(E3a z9H~0k0?U{{qDRPLOzo|k{&vfD!pD8v@sRtBGGfOmtZ4;4XC_{60Sv>B3c-;F$7#9` zk$q86;b@$=4Svcsf%|^o%h3{>oUj`Rw^au6i>D0>-rigG`Rg@NxmDhU1G0-^J7>IJ zB_ENbUq_v8XM81Sa?Q0YQ4DaY;GyOQs7eIDtura@50Cm1qJY`)?&4W#;mPGKCQ!92q*jkA5isdSWwoD zl*)_0UI;L(3p}GsofY3_PWKo@RJzIei`&>OSi0#5?)i_xs`(bm=dXd5s%IoX6E%MV ze+{3I>rqmBUiXJPY z4{U;*f7-@2EEwkY_lj7MP}0Wh)4Fpkj?aJ?T5 z#|Ay;XxcP(|3s2qOr#tbg57dBzKdhg<#v1FxGtz?uiny5S|$fsK!DX7`A<3d6MVkZ8nhYLq<(8pwQP z7ONC__N(mE@5tiU&#!!whaw$^iUgG&|_7<^`_)S?-cq>+&T8nt*s4g7*^5D{+puYolqv(~1 zxRnB!49X^iKQ4}rB2b$1C>82q-vBIiyo05;V{Jyv#@$vOs#>}ijH&Mwn{(^qvJ1#F zgl6Q^L63RYgXUT56`oV^?T1U4ZTk>AxDbKr4=QtM=z>U7dUaHP(Ns#*4l}gtC9_H! zB9C3aAGj3-l-Y=vWPTIg!VnAdG@PEWiKNMiS54P8;(1%CgEp>XD#N8lMj|(fqe%Mu zvd^wG80}98HbP9TeiY>VPO9PHkQ4H;(zA)$Nd`mZBo#$ZD0%5}&?GE# zc1{q-R|TO^A$6Ro3;>AY;~v4j60^eksntS0kcJ21YdjN+={ePda@-TA%83z<$#UIo zq7fsmRxZ`#**~x^D{s5p*M@E0vs!`@{ip3X2TW{ISH3cn(~3@{>9gI05cnNHWn$A~si=U;Or7s!F`1KuG3(~lBzt@B( zD zuAbbOtE6;~yLZ%|g0L5h{qY%QcX$m$vw28Nia*9L>)?@{O)6~c8uMOzF`9XW$Qs4U zA?@PkOX_$P@pPz_j~eLxyy?BjDqcgvJP3kY>6uuaNQcJ1PFY9Y1Vevwm%h#eZAPyI zP=YBF3Z6h&QrK#0Iw%9g3?QKoA*p!ejjBveIsIk078|_a)|gpI7y@0est70F_FP&$X046PYP9vRR4{kj5SwGuWrUf7(!oJ^U?y?|nfoM{yx+|UcCkt&EeXHhJ z&Ol`dkVgNQDAF!*(Q<8Bh@>0Z-_9N}0<#)a6sl%M^@5&yTEeJ`c#$Q2?B(3SBo~ew z%EZr6CbEbZf@<~VV^uAi-vo`A_5RstY|>08IOr(vca9&v*+z5>5hJ!$n(4ZS%Y!$F ziIPmZCl*gYAW!*`+1}Is z4PHvdXO)3NZ6FYiCnhPW$@QT%p&-K_5iF5d{9Edv)%(6LAH&ho z1E1dRZL_TWIZxfFS>kis#62zD+E0x#`rJEsQsUKwkJUutu(1z0dJXs!-FIsd%0X;6 z@vE36_S1*%Om5?dUwsPB;~p2;JLt+OS`3B-JlsUF-T)+P| z3U})@gX7k$1b%BKc(ym>7O@K*r&ct|Ngqrefe*fzRJm)$_v~y+VpNo&9 zV|-$gQ#DEu@LFU%l`c#sRm+h-0})i7O#%WHB4@xphtjdw3nC9T#8RDI*7k%&JFv0Z z6GWz;g!MXx+uYS71`_`A$$lG9X{O@3Ec;>QUO-xHq<-fJHjr~ZG8uO;EEG#zVcg_i z+QW2o=`m9TT9f=_l^&YpD4b<^IBVg}J280aC4%zTSzm=DXDNxCWEde!sylU-QR6j=%{L zT8NT>IS7Hx`n3|oDjj~l*NY=Y@Do zsJ_%3!6W*gm}LOagPZm1!#kvB&1cv1+V%eeMm1 z&FN&K{%1RTX}+s~=kf25eZv|Odg=}gZ_s_8<>~!8jo8?u8V!90 zZ9zCuntaLRNyzo+%Y-rQv)EpTo2+s7QfSBlBBOEXJ^TAO`sB>~mjQ0HZ zpHXxD$N>fJw6hqcZ#w?NYmrF_pc5sM?HNnlQW}thQx!Kk(m6h?-as*_)zTT)P1i#L zuTjqwh%H)OI6qMa2jwggii<;z^S(&&OjX4wS^$fD-#i37La;r%tJ-0Yu!=!+NxSbV zCm}1^A6ns+wC2keY}qJM3P87}57SA@=_E5~zsi6wc{>P!XCB+TtYrfW!d;Y@NIZfs za&eONMpRg#6i~^(nQfnsHRp?qXzBSzy$=6ECNHxGU=f{30bcXMP#>cBhIQ^|jU{TD z^g`MGlxs{WXF2(c{dxx8$2E*WY`Z3fW};K0*y36UbY&@eun3W9-}*+HX;D$C<~?#H z1GzM;*K?+bA6>@S=JUCX?3YkZEKJf$%^8@|p$L)^7GVYx zg>^sH?HkvcYTHVhIywBMds9PPi(-CQ^j>ar@<+wWf20!&g14OxUaUCz_1RT!Bv|4} zlG0<)`aB4Pe)&6z+Q)~WI(3(zH~`&SCm^M9$w(}<_DZ`BQ`b`&ys%S!V8_rr-K5hR zlU0GfB#Kql_J-wvp#Fv0r2E-PIwU^LF4Ch5>LEb81zl%Xv!YrtHXAKF>pZeb%-OS( zaM3ita|I{r(x~$0PQHt5ocD2^Gh@a}{L@?jn7-d~4nJ1!YbOh23scyxJ#Soil?1cr zB!?_gk^S=Py=fmFq5KW&t5^OiLd8SHCjaVGHoH+M5?Y3_=WI3ZJMOukvwTC>EQ_u~ zj&H%(4T`%$t!VthSJRdG2Twj!46o}*}qp|aw!UZ2(C5cZ#>_VO8 ztwW{ei|eo)EO!cJ>0nSd#gnyacO#R`)vLgZ@VdSW)c%V(f4$NU)q);pT%6u!fstW= zybGUpWGpzk!r%w+wd&dAVX6lX6s}DO? zMQ!;J(6?|Xq>#6>hj#Fk1Y8>KRhRU|S?&_!P*lxwgVo+bBO^biFby?Swh&Kj(w36` zeE}!303m1Z7DeT(WwR0P)`(_#K0C{02r6w?b4$klCPCUi;l;bI8L`ly=G69D3`D zra=kmWo9~nAx_L`!go3}BI{P9o-6eph#6V5Tj@}7y$EqV4t11u1#$D~VEG6TYDbiw zmFD_igR6pqs{^EYgZVkZF*ES0i@k_C0xa&0q3?5(=Nf3%peoGj%uy1FQX`~nDX==G z7jnIBzIQ?Ul@~Fvz-UegBz2`FsTXgX{i7Tz&B)|RubUwGO|dm#PeS2r&nd`xo`>8Q zyL%Sk4~k6ps{{|<6J3EKFN7jj*T%J@D)+->ij;bH5LQS7zk!%F*2gSo-tK3&RUTZI zj;Ni3zihA7z&I{VM#g4KF0WDUMz){B2wsP5?;eM2F`vaXVm%%{NT}EX{9TBy)Gwxe zuArmWj=IOp)cnm2r*V{J$Qt^xQTXPaZEVQ`Fn^{nq@tX|xLDvYp>v;sZP+uxB<8tE zSUttix*RHJ^fKg0kbffLNg55#9Gg18%w3}7xKmIH&SdAU{ypFZUPK?aDr_2TV!Tn< zuXQ=Dmfm6NuzAX9JKH+f4-8cVx0!)}HdC6@;|>ySYjmQ6RBl$>tQ(>^xRI`@P`iDdPXAWg`h#H*NMZs|RtQ&LLtM3l8# z#~zFG=B)~mno-Uyjh6n%F!Gv?agMFN*)HV_#Y?gdjt~_~8>P9qjVS6;E+e(QbIP#* zs?czQIL)NTw(YTm`o)POSweX|?*UW5KH;DqDjwzso~E^Cb5z_dyFVxMQ|ASXG9r@W zs)Jwo`pmjPTgHb{A5d7<7u#D4?B};H zn#xm3!G28fA`7xQbGe$(A^>{wd7U@DjTd!i$RD7(8j)+KU2DU+&VvPWG!KC_tq^ER z$=Jt*VN5+rft{sa!%XNmmaG3VlYgA=4^I*jem}n7UpGB`Iyv7%K0bE5CDSKG5AH`^ z$=5IUk8~wJ=UFheyzQ8cuB^M{E^F4@Eq{@0{8Pciswe!ibsY7p zpb5HELI95yU71b^&Qs_*z??cKPD4nF^&#;>nF6&wEMWB#svL23_mQ)aN#}Y8xvW_N zI{GA?sRQRn%|Ergo^WX{2Ef>iQ)(yZ4n8)=K#hUZdOb@_55PV=m&TzDo3o`JlIv{> z^@PywE*lTdetRQH^li1AHF{#pNmhNl`c~jGJc^*YzhdY3{L%Y2VYH1Yy(e4w$(Da&{}UYMn9C*($+N*rLI&4y)7sWPSUb5g3z%;io)!3ftlNbQOG<+(~{2=AuKZ%Jt^A zSs}`@hkcUZ=A#84k5DDY_`R}{0WI>3xHNr;6sWVC7k3Ke;cu!>unxm<;Zpm&@~Q#P z@WuwBs^~CIRt)aSXC&jPbJeK?XbAN6azf$h@GT%`foJqas=!AII9dGipW*$@SyouA zdRKId?6iHc{fT~^Rid+0O`6J7Q!qiBLcMqEp$cp*QPk!Ri^}+66&R*lV>L0kl$9bF znAKf28F2ov#_6enN3SQ>mWOdwXRM%M;wJLgt=fAr(DRwK6jz8oSA{zMs7~-7gFs{(RoLe(jAN z?)bib+yo~|If`)I_qUD@VqSnLj%`*GT8#pP`EDP0it8hQFhJgLGGdHasw?957GPTwrfVWOhF7`_wdOToQg?Ml`>t;Zv zns+ZfXl^$NuAy68kwjEOvL+k{)4Gac#*#&v(;y5m+M;i=-!qeu-?E8YjlJ;VASImQ z-j6{*^x@smN*+x^i^}k%9y^-o3JeoomgSUwDja>6FbppcP8M713ZE}!#Hu5a;WbVv zZ-bLxCT^}B8)b;0vvEi*m)uxV!BjdTZl0EqE=oXbl5MW3MOHIBvr|dt#id%4t17Ch zpy8=6@uhuklzA$;drJA%dMGYc%LPECRAs`)zW!4r#!B?WY9wfc5VJS1t%Wxb;)>NX za=X{(ibJl`cj%%w7yoW;32U{Wa>yH#`5cAM)gpo=3Id@*xoBHyP~-%usloBHX%l_|vs6-KOXCJtQrj*MN(JzNS(GGWZKf~q}oz7N`kh(ao0of2C(8XJ6 zc%&3}XP@Xs*LL#ZR}j5IBZCQ63@6nLf?Vz7pDnX0w^(Ds$)R~|#Wo@&cmrmF%r4l? z7R;c@)$~=7le{P0rrh$?r%THPIY1AC5 z)n6dGq~X(CJ6N~?zVs>NBC(uY(J{^yC(pXNO&<@FTm)kyu+a@xJ2YLy|@NEWipQKvk3#ILb_|1G9Z<29~)k@53H45HJp3IVt;@xN5YTyRXBA=k5 zBY0n4#d?fjcW-q==bDq5?C%e}7}+I%^-`h|inGwg{(1;8;j6Q!;E zkv|X8^|uqv6LCG+L#8 zqRLNJ-B{#B!)V+s1OEZovlt5dRva+scw9q@>St@>91XQ-vlOn{OiZNp1ET^ zPIkM7qeuU*&C-fJb%Jn8A z+}TVy_P-rXC%GWlRa+ybO=YzlMnFA6(?w>|0_R?sE5*(G(rTS3RFKhz`TOq3C3utwsw+O^h%A7 zU{{h3gfL-g0z-gq0CtySCl5V4L9qX72mtMv9WH}UHKR!4GTf{#Uj{((WS)0?jgfCe z?`#Tl3fw||ubz?nS*~S7?!PrJofR?gk*p%#{)`gwg0UA|=|2E2ZDxOO%Br}nVOyQ^ z;IDt}1tT^-xau)d(_3ZqDpY^?CytpTkj@CrZd1M3N%6LAe})eSwrzRj@AK@rI??AL z_dT_2c3x?vweeEQFI|u+>aGgNa{jQlRgAmqQDq{E8d>d&>|AU(ZxI1i0j3Ji1Ddy7 zg`ieSb&}4`yvStF1(uh4xY22wGUirBuB|ZMBYy!4(RniFSQ3h#m{qj9^Hn66MKfW{ zU4K0EI5&S(=g9rg+-B?fUW!aFWKa&0E~dTRHa~eW^)ygR+;~ut`=wj><`>ZeCNKKBfS3L3~saxkym+Nktjlr99as)~7#u&{(+gMx6(<}=1Tz~b-5YWmX z`A4f>;}D!cTJ(ox^xh)%%Pw<3C1NtJ_&IZ%fwh>SZ3>m%$7I2UWkSCT5)g)1)c87W zutEL|Dyp=CU$+EBt}-rKrK^yp>#}jJ3UY54KAJjxFp^EWxbzU%F0MUVy_aS%+VXvx zd9dakLi_Q+my)nl)UolW;L;6|2G16!~LJ!}I>WF5NM0-wIpe0-d+wwayd zs^?(V=tnr~DC9UNrdaBY%^p87_Md^zP35`;`=MQf^yo|ao*BYAw=503)Zjf5?%svU z?tVE=EfcYgBKgKFO3RbOXH7e`X_=gmzqWDyCAbwG&)dK!T6WRWxS5_BX3F5#FWOGg z@7VORZIt&Y?6kd?d}?i{2B!hFYbde_Q0nA*Xd)RW_J&gf;Rc}~Li;<&oOo4uq>!jy zF9_Kx+I&ny)wuffB6z*A36)2IxLRePrO9D=boaLnfBqDVG2zesHqjH7Dc zeXs1)R*78nvS?Gr@dh<5Hwp+ZVLF3N7J`%K*xFAZJ$j6h)TzqH7hzuZ;jCe2GTVs5 z3xC~`aM4$qUMMDeLDieU?b?Db(12|KbM?9qEuLrd!POzIQKyi@HauJD3|z`nSxOqE zOqx!L!dI)h=gl5{@Qf_RpP5&ObrWG})Ib@QxEvH_sW5=u>xJJN7R*#CUaK-#CtFhFuw49%zBk6|D4aO( z%?HZKfiS+|7F>tG5VQTI`LtIE$3t4I>$^p3piz5&xNT6i?Wyp2iPUV7%|-z-waE|e zrEQa^D~~yknVi%k4}j76col0--Zgyef1+F29J>bXcavRxTRMWz z(P8u1GDx7`iRV(yFp8kiP!T^5kdc%vC2?w=Wt7t4x>^>qRDV|BYgk5Nkz=WhbEzY4 zvShl3ChEY#fk&twGJ3;awApn~66YlpY@cq1G_@pb#Ld11E$B(AEusQ54+m`07c9pd zXucc>lWEQZgkqbWZZ2zVuEE!HMrn4uv7V-~f#3UOrV=fUbygv3XZRTVWgZ#)K>A z`WS{+FyC83JcTW=g1eTdq-qh>V5$xWHp3bUUkj(yc%}=!w^!+iCX}l~9d1;82dNQ% z29dho18AuQ<=% zTHg-#ZllJzF?|AgT*$>mIje{a^K}$sz-6jcP*?C2wyr)Eocy3 zbUjzhp)9v=O8-ZYig-8ww;^lAti=)mD*erhOQ>ItuoCq7eg=o?W~he%Q|LqOA?odVDZgWN*%Z>(8#aOu5;%BEa8Dd}E$`5GlqyX$wE~IhL~y z7(S-QhrSi{`NI?7uWS>s*5mU^?1-jyM?EC!&{HZNw7tLQo89%0DxE9I1~iX+7laE6i%$S9jVXjEkS<{EA^hn%0IEa}1gf34VBtPxFjL5A=+sjP3-fmZtXe_a#whuBx%uOe>1E7gz#1IBAwtQ*NN5Ge2Om z*65IId{lqc@!^?B{jj^%^IFLTY8avU88IUr0zyrqL!S_~mtU__-e+~NZ{3n|oi>C+ z@TS!Fkh$viuK({Li7gPjG?3?Yu~(#lc3H*JVea;GCk~rBn5ur?fJwyEOoTq568tqY zis(9UHDmVH*cAKj8t|O{DV08~da9gt{FuT?l-G%R_C2zfgM&q-4i+u9vfJdOzn-Se z518>j-RZW$^ZMVg9+Fbeg|5-9i!~j<%mvoGF-DzM<3X^KfLATeJD}-M_si(&+_=nn zy!mKIy+p8A*ZtFsxz+5$`$KVPK;={4o^kScP?btZI2#ba_siiP2Zn2?<0Zt>dT&k@#rqqeSw=r;~_oYm7cr2Y|3pL%57fCZ8*y9 zZHl~ViaZ~RK3R%BT#CJG^1W{Iy=d}1UGhCz^1W}ey=k(&Y_dI@Uq@%MJvg$xZBl^+ z`P`hb=qzdOL);{HkT@P`!uG_$M>0q>y60zN_fQ++I&X2p^-ECCn9Eh?L0qAmKVZ*T zYm{y~&~EYspLTl>@~&}NAK}8zaF8=}`C6mb-h$d>vD;I)+XRK{1-iL+T>^ydzA$H) z=sR}<`W+)aJh>|)wYs<+u6LodD04ZjNOKJb0`1P1vn2(~VFhSbP9f3LxY)dZUvaxQ zfJPb(Ss-?o2^LoKNa2phJnmYxZ!t1c7$(nsY2PfBLK9rqkt|U5=Yqn%TmOe!1vX__nngO*3O`$zHG64c(*F+tq+<^dpKwp z?-zQD*OpJJO(tGyvPD)wI8o7?X+RC2t#hu)U+O(qZ*-6I%gs;bh}U6^2A2ZX|C zbFME9qf0y6m8O>LG_qR%X!ikWzm(--rkP{VCOz@CXKlLuW7LTIKvg5Ow3}K8<@6S! zgc@ENfl?c}Ul|TBk8G0Csgx&K2r2#h9;^29o%G;Rn!)zRAKnk)Qkmi=h94A2KPq>Z zYGjR~H5Kg{0jq_iCe@N-cxdRChDS(fsI6^yjK{ByO&Rc7Q`_rJnjS*cWbd}|`E6~i z%UH~HDx7xFgD4_wtBArb2`Qz+SRB^BpqWc2TkeWQ5Wq^xae2GUye94j*lmYHxEqVKOBOx1?y3|nB@czBT)b-<-Ul|9PecAA4IlkVd zXe%CW_{9Iqt-_?U`pjmgX~UOl z`&zW~<*a|>v~S~0+85wo$yasFo6T-+IC-I)E@t1;ue(QDSL{cT;SjwbvY!4+yGQ+@ znNE|lAr$`G#koxveex2{#Cf!_^GJRBjfRGf@5B?Sb6wSBhip%|rlOG13F*Z?+!eN8 z!t=xwH{gosSAYakDBYN=63?66)9dqSw6b!yAJRYt3QGj3{hq(QeX{lVAL{QS4ZT}= zg)LR%a0L6y1X$7|$q{u!1rBJMD33^=CgCvAP}ArKY^7PXj)jVdah6=LYowNSOUvRcQH7QM4KDd>4z+6bUzy>vmZ>7rSS-z03EWYZ z{rO&Me)RwpRX)X;B_-D`VjHKD-y+T6MzG|0F2q(Q>4G)CuPnHSN`{1j0+gu-SyPtW z^QSFU&7i@#`|FtAe>AJMBXaRC*?dW{Y23F;P|l|bz_+R9sU@Ngib{MG(nKC?k`+)g zSC`-`N;zE1f>7ez@rA#+^HtBw#m!8~hp^;_j`srv3O4ZM3$C872tCPU45PqfG$BTT=THK+4ENqBL>d0gQK$UmlY?%_(Qh;5ypJDF8ropd zUx2Otac81v`-9xCtE%nV=D0B@N^r0AIiuf6s(?_@U{HOJlGM!kOB(F7C{Pm zr}YuW;0ZFQU$A5JX6XEi>zDwPzi<nE zy1!JthFpgc+fc(&!Yg$N{cJRKsjF0ELB8_9$<9vwY*e)1rj`u!ix zF!RPQe|E>CJvfynCtGY=ZkKA2}iCnCjgVWB$;UHR}aT84of!~JYXW>70x zy538LM^1lloGtV}PTQG+gVQ&wHw?G#E3giBJ0r98=Q;$2AADf?E>0nlm%tbRmp6Yf zsM=u=74kMLpO7=Qx`@P!(g!Lql_Fzh@bp^1;iYOCf?~xg!lH zy%yhxTx4IyffkL(Ny738H;#eUN)5vsDkHxfwk6Pjk3U?|qR5MNf;VCwJYLtXUjfj*B)Vi_4vH#DA#Lx@abajmM9 z5SA91B9kvHqE$ifLold{)Q@+#ee4fNQ2jYo0Kfo^oxbZ^$og;e+d=k`|GOmrCr>J& z2;tx@_JG&hj7Sj8UG&Gxt~p;!huW{>D2`Lm(_M^p&Uv_n0yT9ls?tahr$Kz2Wf{g- zuWUh`sgvOWKHsF3ASh;&^3wYk zl2WsHM<$X_?nw@T>LPq(@bN5)C##dYr-vec+bV)y?12&oS>IB2ocw>Nl;iVdLCn-N z$LH=;E{6B#B1JFL`}+t@FZ=84a-pHe=lLyXb}f7P6embo{qyWH5+eOD`=qe{q3z_x z)Fkoei+^~81Qhl4X|Yj@iWqUZWb3?>)t=xpt4AwGrsvbi;~2B{|H9GzNXqJ(HRe5k zyRx2pOR{5c<}_*Eae#R?7nCKQMOo8BJYtlp1lRYd2@OxD94$usF$ zucM4ojZMwdG_T?b5IiJyV!WbT@N1lYF6ZiTplC&%3<5YAaJ~#!%I-%qXta351&ZD) zPb;R2lHKmA5Si9=0|@0LXyvMSGtz)5yClX4g|$!bV$DB&zpzwgY(aa1WoluTG%D3& z92rs0O|??}t%@dWth+iWS`$K$1t;*EgrlS1;tgHqDOm0b>$&E**T?MBCW6VytA!Fa z{f~r<9T-U3SoTcww^jGYV&}%I4G;0;$6o=v_B(IYzbd2VXL~+NECFhmp(h_3w|}N5 zF~)MbIp2Fq-bo&Ne%Ls@L>fWFnBsA9e+=+!yvKQFduHx(xLhaB-!$J|N@FMzwm3`iECsV_4O za(8X2DZ_0iypEfmh%%3)tZ)N>K8tKKP6D&^biQxboDWv6q9J9%6)G4vy1mFr+=tl^ zs1^IqRZ}b;y9q2D+`L}P*`?yJA*zXJN zxU*>Ws#k5=S72!DvmEt*b+eU#*f{QpwwZJ#Wvgv8cqDA&yvWhn`)hDA6xOzdd<~~a zhO?vTm|IjCWe+dVB1!!LMc3&QRHueYu;#pxwq5>kQq za&bpcFRBrI0Wt4_Q{8Y?Sryd;_mi&=_jX9M1z_@XH zOs2YdE&&LC-z!J4hMW_TNhm~Kj~Ug3saGBT+4L3p^j3IXGH;;6N{=&mefA$T0?(2e z)zwt7EqQfo1nm&u4a>o#;vH#_F?Lz9Ui}qzu2UvcFf1>T)w_QsA~boMk`p(7?{l&w zb~%k9p<0hUv`90Jzf?cZzX99x#|AM2izSO2C=L=L$y^=N;|{h|kD2ae^!;@}tXo_j zQO203=>NVz|E7mJrJK{Fl}%k1HL&LXgv`A6KsWD^=)OFihIA@x`wlEe?mlqHm^rd3 zW&;52cWTD1|C(y_i-9<^y zLBmyz)>1C8HnvL84~AGii)m$+|4K&=~^|!cIM1%%S{Gi2Z;R#aI{ME9fLyJu5>2?!}@7Jr#D_Fz*N2p56iU%ZV@RM6o zHD*F5-jLO7oiG?Kc_B>w61GT5t4i&39D&3nEvz?=#7vcV%$^HL(`G`liVfmrFVjZB zIf`+o#;awnXb359Eh%bU0ibB!ew^fa4QSrL^Hjn@;E#vu4{#w`uYNYuMe0|{cQRVs4;x1E?PEda?)pW(I}DELK?p#{@n+z{4f$D)QN zItrM2^+)HY0>qU#X5!%SpZ5IFd{)E7`ubp|=V2rZ3bBbw?jHvnBxJ;jP#k`+I4gL1 zbv&IT44Kk;_VDRhM0^UBkUz9*fhQy1yPb}M;qf+Zq{SpOFV?T2XyukJ{@uqQP33~uydM*-wl>3s>Bdffw1#07bjeL#Gzr|<(y#0`w024*Ig zRx6=$qzpu@g4M`W$)W2p0XF=03;VCTRuzBzM_6^IZE%dy^<+^DvqZoDI?fSt;3&pf za^O4#FepCUhH=1ut`e541M0P47z;kkF2l`FlS{7K>TcBGA0mH;b^T$CZ}4`px*K zbBA+l8W38RL!x{i%oe5tL!yTCR*DT09(YV#R}D5dCCZvPrn|bjq*9(zCxQd^3a;SJ zpsmXxPf)9=T__3=zeQhZ)uUmsOyaN{ECdy}snZilgwvI_0)~ss4$HWsE<7&Fkb(H* z)sUST9C~MIipAI(%L}A7t~|Li`bwGz8?J-GhqxI{6H&@WzpCa?uQA&oB&@So^J8==O@(Q=kj5- z3ANQrrk;rlI$0o|D=yPpwn~RyLDZUpY2MTDC$X|0BI#J490Aq=uJKnuz{U(rj9<}r zh9j2yqJ3c5M6=lAHlo9cxapC|<|%HMr_lX-S^yL&ULscIs?!TZ?wd%^yvMUnH+QK*Ovkf>N3+y* zat1&!E+iDvq9ZPnGEw%_?82z3HpGQve$NlV!V>xTpb1EtJEN{=Fdec17(>T$+o$2CF-W3aAE`f?Z{ zg{$Zo&ZPo>(a^9gZp^Sk%j}LOW$U_NrcN8>oL~7vPB(_dFyCJJ-@1{S+EE##0|xDz zD+tnQ)!X`?D**9B)Z1remuZ<7_k-5O^3{w%ql)h=5yThildg;aggVOoJJ9>(J+a0O z(X>?S{IC>S@}9JY!GNq+=bEbwf$tP!f;p~qu;=B0F)Z-nng&iMdW>nYT3^Rok6F^L z>_D5NL|_}%_(O3XE*P?|EX$X@Tn^ybFEGEFOUCn=5e{Xa$r>YIp{}A=2CNm#YI&ew z>m~B`Tqu|~S%X~EIUoe>tD3h;)huzRaa`FKtjDILSH*FtGA+v=d>icm(N2B$+qjAa zEok~zqkc_)>Pnv5%xXPEl?kKdEHe>{tu70eLZ7TQ?oGLy+`10qSA$73EsOu)Y{pIX zNo3H92ywR>fe#n8iUdevz^yys_3n5;-&x1dBD$rTbtE{mzTa(KRhQa3cN1mo>0X~i ztz?0~uI44JyWt$l_UJAxFb2aL0@q!`1l{Cjwd}tOlgNbKI&GQ5YxPcK>i(@rGtS&W zaWAOr9>yh?NA+x&FqVw!M6Dt#?ykMsso{guACsi{6S3Sal>cMnY#;5kYOqJC0{Z#6=2&iJIIvjj z)=Y0?u`>vtW*R#%vw>DsTuxqgMM*bj-u?{e#u{D0pRR5!V`9!_ zU+JGdr2CE1C!G(Bre{x1MGn4D9=n@sH&gKQz&BDv1!byXZBW2u;UpS0U3!b2r9dxg zx_BCinWa#`G#)jF!NOS}YO3rQIbHh2|(aEZ&Ryx)5 ziRqXbyt!BojOmCx+RnNe%0v5fl)jBqg4J?`yud>~QWV|=x>dvGOMgsYY}w(3R+7^w zm~Sl^Y`x7%q=G6I9-YasgjT8zyAf9l_>Y24x%ZO$kgXpP&}{KsGTQ!R zQ=?H+wfyn;Q2LbL)_Ric9O!2KI>?jWQ<@-B8#@pmz=B~=!`MHb2oqE;<$BMq7fF|@ z(=YhSSRE(35A~_($h!0_oGA;JqyQTE9%?&`Ut>I+0?tY?U%~ z6OXhygh^{$z00W;reAEO*~8o1RzoNV1lr|$fLj5U@`w;|3^=BlEZX-Vo%_U{k;SU9^ z8(>PlaA4}~3+Bv+9Ni)WARucQGGZcMlARR&EjN~>Bwk|669R`8d_ql=RP{0!O+YJM z$|``XdMAzk%O8QS>#91@*=XJYU2qj;npum@367_0DNuLp|HIfjMQPeJ(Sl{$?6TEm z+qTUv+qP}nw$WwVwry9P`uaa}F>B4m+`JhXnHf)Bvn*d z><#gE7ZTytxQk_H+c+=@Kv*5up7&Bm2w2}!JKU%wJA-UbUdnwK=ZA}{158&P_`MA` zfZ?eCcBgz?jv;wutI=r3BcXRtXY?&Fy+_Wn9VTi9FIzCgK!4GdJv?HZPkw<}@Swba zU}O+>azpZPc#%KQi_WYNOr%$xjKy3KZFEN)*s{L@*N|U7faXI5Y`!&OBm7-`>& z)6|iJ+P8M{FHdsue-4V1UJT|UDSp@m=t{FnbQ%U`K9DN_A{a`qRpYK&XiA_N@6z#nl^dvRfhuCM{>EW2+GUn=~c+&y3_S zBfGAtA5ng7*mm+crHps;DJ}Z`)I?G-b?n%WeB(Eyfp^SndC7>Xq=c%5RPaO)Pu*@< zaTqu$UX`x9RgJhe7q1oM4Xar+zQsS3dJ>!BeDc zzj9%r_hWy`)b#5}Xik|8q80{{Q^Z zVC3^ECiDL@;>X(mrnmw*d_RF#i!4L5T+)IYZ4<{V!k&v84W55L7`V3kAhQYVQ87I2 zo6Yv?C*m13PqUqLS6`7vxLn^{LD!^~5e&6dr(mnlKGB*e9ADTBU#Zl+b#p)5`x6d zE*h5nb)>Y->1rTS>&wID5}3z63DF0YRb9u13d)&8j3_iZAI>gI!Mn(q3)w5)2VE>v z5~8X5jQ#s3C^ElK|7RFZR09`VO^+|JIM!D98^OoK4D#=?=iCw}4Gmh@UqHgDAY4q0 z)Moj-P9FZ&lieF9R&Cs~iA`S_Hl`xhCE!`x5bM@R{0e<2gFL%kb%4Pa8o;1JAq5s} zX*7c7rL^J{EI`%D$bORx$N4@oKv%77_ewLwO^dsOv{SLlYC4Am8PMJ7gb$q%N6}Ld zed($*3;b4{S&c$pkmIc;)vh%B8{ zvZd;ACYEbkg|I1nxGJ>Vr!y*J3&P@uyGjQ#2V_9LD(yohl6lj}%={iZa>#_=p_AR< zWLAePQrlg63jQEbqYi}<+8Gqx5*FTY3YRX*MR`Jmy-^4T&8~v%`Xd0p#(xtAfQc40 zQ=1iy948l`r1Oqskx>;;Qn>PLd`rp&&2_i*nSC+fBH`v`}rB2BA4s_W+D^G zB^J0OS0GJNGuh*4F_P?2GA^9zf=Wshk4N#>+MfLYujLh~KBq`>O>Prs5|Yjp1{7xi zbGRBg^2WBZ_uh)|`Z;l&1F%Bd)>k_P1?}x@i2iY0h~HxLX5lCZpnB!YnjPWr(HRJ7 z^0cX-#r#IJv^mXP$Ai7EZt-Cx`<+sy`yMYb5J~Abc$9JOVEeR14N8YLetwKgTwaG* z+~rF?0@=>;iZIPy5FUZ-IH>q;BUaan^qSyWr?Y9+1?2)Q+2C762y22;s;^?F6EA88WQBM<+s+ zIU9HoujI;ChS4=lya$qXpfjCi+x$Sl7$zLwtJ5QK4hb9W$S$NV`ONwB*Y} z1vK;JF1gN97u&&iTjyUAmTi|WEa!O`ML;=ZXz^?SkReOeRwPEd2)W!&?PXom%r)*wo_#U~sfCJ6v z`hV{bmzh+KvbR$X;pXcGPZlByAYJ+s5*;SzAz|br=h$98AZ9v)*;r;b6dfi34HSPs zswl3iCj7lke9yh?Idhg{?1M0MTK%n>?vT}Dhfk(Yn){A5KMVNZ!|*YODG;?IFQ-pb zXz2It3C$#ssI4C(b*jSDTSa4(_$(S%q$??5Bt@+Q2L>d*<#UP@Z1o zmPcLogif_s7=x+tbBZ4{$ccWk{2Gwcc$chyR5_F>X#@UEE}z{x^#Z|Vss7PcdgVe2| zpDa_`g`0phDZ|^0`ir#HGrS%$dxT`Po~J=Hmr~qPp|GUVKTC3>sE7Qz)sIb>uC>1E z(=yI5m#4Pao4Y%MD>nk2wdTw1@$PO=Iu`4}A5Ja|!+a)2yi18t3fA54N&a6I{ZolB z9P44aIYeSWF+gW#UC980`D8m1D+xmg)}7$%wiHDFoE++Q;w+`9LgA{5=2}OD%I)D7 zYjez>7F^SbTmLGbzmq`Ol@5Kh7^8+GpytoSow022Lwp`Sn;YBA-xRKNGt?-tfwKv^ zbU8Ymt!*DicYBLET87ryi><9?RimI@lk)go`3wC*YIz}H^#ZeTDyA=( zgQ8k_dA-?jk`7>sybKu~=zcvSXzwd8uQnr}(-b%N?DQzyMVexlB zvKg|Gq<@g8w?b%y(Edgy?uQDI5VP(ju8CpFq6 z$BB-{SN{IE$BqI}Wy@Dg#8k>(_NO$)ofoQXG_v1o$Nt&&e3GlZFGczZnO%`~^rP|N zWNP>HzER##N=!KwP!fB1JDqYh!q;vPD4G583crqIY14V#-pCMP7c<_JFF_g916;tN zdw(j3tS-Hj)-NC<=3wZ_By~e{n7o|-4}|Kl=S}tZfb->4tynVNWQ-7T5u9uAIrKWbiCY1{Ez)nS`Wcpr8i z9*(!uj^phuE$t71nC-8D@Wrgx)1SYmgS`|n{NZ6;EATTH9a2NuIrg-gwDAa^N@Ss9a&1r{AZ4$|)QHPgh6(cqcX+-UR8XvGE=29khq zz@indNh*F{_*cPNM$YxM)_l^$joYy3@g}~j6y!N+EOfU1<(51|0p}oM_0PRNZK2^t zx5F^Ml zmdU#+21`N)QxWn{h~0(NSgt=(NvT7EM}*ZcyxAStZI?vHpb)hB=p_#j4h^C5yLnHc zg%Hm}pus|~dMm^#TNn%x|Fs7uEI7CvPiKl&FH!+fCpki2&ib#^J zR@zv-Xq+3A9ve{kpcYGb6EVigxIpUtfunq#nmk13mJm%=l6lAYLp`BzX8Jw7HJyW2(q z`kvr$2ZNnMQFH>FZnVYZDy!n!(rM)~)|6eK*jz_x;Qr-wGQyUIZ?q8mqve;;XDSJgsnVO#_J$vjzzFWZs`~mv# zWHUfps{7@jus15?qn_=p^$F>{yFXr+5AP-zk3eNjnQ0@YU@-{Two;ssyJw3hP`pT> za=^zSa!s6*`V7#pBQ+U=X>5yX6)uvvILVT1nH$WnM^%@X2xWTQ&VRp>ebn^h{S>EA zIXJkuxwaT?gwk%}$}9m2M~gC&u<=hRL*R^v*`;I=l*-P~OKLL|c{%zr|lnjONerFL8H46$|aMGR;P-%Rf#NtKMTa}7h$ z#|jVhjqhz|2C>U6g``lFctrpBiU$`HP|J%fA2^EJ7AFs3PO)Wth=9awuKTaH7~&wt zO{UJ;TDW?)e3;6sDR$Thk^pEfKjN{ns%|4K5>jEsM=$_1C5N)3S9r~(eEov%UbQci z5~$xjlRD22XU!a)HNropXzxtV_qh|8_6+{BT0CKqB$NmI*%n|c)vXzWYh^?)vmrq zKtLmh{dM3atP-ddw6c{^W@Xdiwx>QH2T_h8YAVfM?Ri{wuWrxFUISgRNje;o$;sWv z;A_n?FPGmzX{utEizR9xtEhZP@guTMg{p9~eTcHRq2W$r=ln$DoN3n|&u9`#WWHFh z(JTfcV!7__uClkN=5Zz`8qcg-G67L1FklN5|Fm4pGzb2f$iIk5M^Ro9HSxrmMg2jw z7Wdin(G`*wfJ0+7wKEt8^u6hP(O++FO!b3^HLjKo73=7IWyC#4qnmunC1>oc!mP5@ zFd$w?bUIr~`ZK-UeHureM;C(MwM)!ZVOw!pNuv>rnHyb0grF6uDvlzISQ6|)#l%7Q z2nCa_Rcy5c7e(6JsY(|(gpM@ce-0bii!oD0p-slX_-n=DQ%w5W>EB$IpW$>lQj}}X z(91-Wj9QA{1_RqZdkY~YW2Htu$)z76w*#g{xRasnt6EbqTQ=a2f_@zb92Nq1!lQNw zn*q9@GKz)SFA?iJLxB*kKEw*wJ38}<6mo&`3dUi3;hv}yjD+d1tM``6?B!2B#zQdD zAtYh!TSY-JRFBb^y^?*{f+kA>C25afmlpzh4$k-Yn|m+`>q2BABqH;-<(%^|7>wyM zG7&+K6>_2}&dp`wv5A#ZOX9!M^;U=g0%oko-JA`rd$D2GTIJ3qh-V6yrOi=(!2FDJ=U)awLL;Bo9lS|K42A+GN7(cM&XpPgdul zrVz@wv><)OU%*K4n#aN0nyqc?whE6yOz`{mM$>)2& zRX6nME`zC!N)=VDpi9zD`*~TPNZ^O@N}^wDXpYZBBG-*LA6dxKU>CML16d`8B#_&a zJkKl0$D5JNNNv%%xzZ)Va7 z?~g@v#C)lt;cKp8iuT(=5qT9-l%Mz-Df`S{9)RzHAdf#hS9>5Amc5oHSW>u^)T%ug z8?oFrd$$}M`{IPxx0Fjg{@^OdACs zEGXlMQ%Y_)#@L*+QK)Jx^+lxV-fn&55AIEhi~tr&meWJ0#$T4vVw4y}Jz&Pm6IW9(eM z_kM5hMK`fOA1Hc4`0Q%#G$54T=p_{Zlimpg37 z-QiWm_}xjTmD5E^l9t0Q26rU%RjOx>dNkPq=Jnu zhbw_x@Z6C;H_M^ZsQM?N3@*Qhnf2T2*vsd4_?~X)3I`qjmNRYKRx6 z2N+c-qRhC0D$Ea~2yWcR@RqcX{uLa>W-AL{PO|G&WhYSUav*49vDDv-(nN`ngNPV` zD&Iyp4G1)h@1Ag+MdaI(xF#(V#&y7lV2mqYgj~VOIq4)G+L|p8J^%7n`&<(!Kb%;K zU}MB6-Z8I5NbwsD;{ON6O@%kYoWZ`H} zqIygN_Dk!ASvu-*5tpEm;Q=T}{I0|Nt znHd-K5t5Ub!vMz)!uUUMX)K~PEp(8=S30idIeZdC0j7K_nyFD(q)_0lIvU{yf~M`f zo0_RiEhPv)bmfR%+2BPN9&9PK2NJpzQVjqCHWoU?*$nD#&{{^c5r?fRAQDzL9x z)Nd~%erKrT@27Bu*os%nk}jGT3xX13Rf$jt&$qp|_>oW|-yOAIwFOAU!=EVHhOFdn{K#wkQMx;@*cN|D{Kg(Nk`%1p!T%_} z==ONPmXys9nxw_4XB8N7i$D}LR5l|*Ud^2OFMo>{QHOGQIX<<>_S}s0 z!q0&e!%zZ0yU}Riba*izSJ1_WsEPfMx^c)xAFoN?a-J$U#M_Uzc5sQ zu6$QoJv7u}%LP2c6yOlyNS7?dfvZ-XBoT5OAA6($l!cK z5K>EbC@+d|*K(&*^67lTNZ}bOim^L2)cSvmU)_U0*;7wH8FBXKhFc%1=>PTV{$Jr7 z`QHHdZ$?9{cK8!;Z0jGjO$iRQ0v1#E#5yP|HNz8q5egX zxCd7ILB@2;d_rTL)nu)??r?k8mIY-0xEfB-!|rnFhF_WISb?jfFT|UE^N~qWn=m>s z&`4aB!>=h7a-BkMl>6;Dc}wg&T+`%nijJ4zFxY`9G4k=~WTwI@T0cp+92-Hw66tuK z9I3yOesT<}k5{{8TfxZIDBuz<%i>ZN|JBr#5iQ5!DMQ!ls^8n=44v2vHIbqtWZXz-10T+E zq_58+R>kG2`#vb)MnTpEeDj(R4{|YLCZqq)jb?%DVR*^%n;w=dxK`6G&orCQakF7z z_^?f2Mj~U(&AdF=?lG>~vFj?TEHiFMb+Wwl+}J&f#&}Wo9&RbVnsO#$4gDk&gOGH6 z77J>#wor!Ao>mN^^)P=@tDyD3Zr!SU0}hs3RPM!d_*2OfL4uM|0bdQF9Zw$=>i`(* zU!WAjqENk$FBRTsX1hz42Ym_c2<2FP=Sg7?c+r z<+3Mk=OW#rDr>mR~9Ll71QxWI>XMU(vl1Gu~+s6?3mK zB)v1K%*3Z#RM}cH6V`JJJ23X}egn7pnHF@D!9kSyqhQ7nK|D=jfFHvSo;aSdXz=CYgGWeChfcVclU{>3B6KjEFHPJ66nUrzT$ zV4E%gh||)v>C@nCi;%V=_(U?>_z1+r@<(F6<^JXyZ?1Hf$Nr34L*KL~SbqY`unMVc z9~S&JTdd;x?yC1rjN0`Fs@- zxf0UL%<@L*N;zGqSCY_D85G-zU3`mfU(H`>h6Qqb@3-A|fgUsLvR4yj_*^*YGS0?d zY9c$olzqhW-Gb3~^&Bjg%v;ziw8p$LknYIL5pneK?~(+&yl?ReM2CIX)$)9t{Tl6=_TKTKqA*lJA!1&c}Wy z3gJQADlv%JM&}tD@b#{TJJs|+L{do0D)u@s>Z5}P|;3E5VcN&IH>Vx8m_P>aKJpb(rK_R) z?jg_>L8Ey76B{|rG?U71s;j%Y2D(4B#Vg)^Kz-}iRs*$Exuep{N0l8<(H;)fXWp%j zUJJ_q=2T2xO3my>gunny0*;jwdNyDY&A2LPY(R~iym={Myl2{dlr`6?4dL9rEBopb-AqQ+ zzT*3oce+(Wgvxtlt*2Q(dm*2`Vh3Xk%ui)UaS~KqOhOGQ5l}Mm20EaB*0={S#8-um zZ62KwtBzK?&}{Xi$C;v6+?x59Zk{Lzi}FrD5Qydk&H{yl#x^Qr{3&js`Vm$|f}S3u z_AV&DS`tP%8=(|h-!G-jA)}I)&qGDJ5k|+tV^^3~(9acA)~B@QYF3w>0A#Did3qH~T@js{IZ(G!C{*9l^dQZa54=0A0p+IfX^*%q=q9NXPqA~pSicAxdUz?44ls=e zu@f>GCSdHn0BiCGE)g;-!-S->n28uLj_duX$hh~YXV5MCCd7PYpeGc%*U}5o0>ul& zo*MZ>4diT40DetDB*rmACVpC~LM_{zsJ9k?A&~bo#;>M|A>-h(Ai@Z&63iNMVJ8QD zhyNKkC}U6P?PFH7L2En=iR98%llZYqYt@x0z ztulw(1AJN>^D{XUfq~%xTU!MQ6sCcMr4m%qCSChRrI-gGF=a%44ho)@7Hsmlk!6ne zY!)xvj%*OU8%56K-@Lui&wui3)~$}ek35F)Mn}$zjvR>Wb=-vsjw~rW;$XJxVI>`v zk9^9LD~8p`jAa}g=w1TkvHgY9!# zxnP+D9a1HM_r2Xal;*y{ZHNlPV(ew?!A|1+O+|?~oJxL8 zE@_pm>!s^YMf0wMk!K(7?R)pgOn8g&IU(2PW{PBYKqCdGZ;p`Pd!Q;J;sn+nwoQl) zaz?t^{d=2{PmudMOr!p3MTT<6fP2JgKHTe|SGBkbU&(<;Z zc{n<&;ZVmk4YW!54{Phq z6gOl2WtR`+DV+~z(R%s&ZlPbxk< z+4+(x#lbJ5)C|qj9Qf#tMNA}>&(nUNy=gGZ9?gwPl12HU1AVc*n2S+U0X5hdIhPUX zF>IvkW|45!HWjbhCHxw^b4(J$HkQT;YLAT*Ebs7d6I&oNpgl&gMFo!@a_K<`zb{UF?|e-u!G%g>lc zky-6xKF;086K|Klf$rqN(%PZiR7unf7!}<8NK=yz(@|YW(Xs#90P8hPMS<#=nGnTo z?qRCSK{#H3YTHNlYncBHivueYplVmc(nr=WZ4J!4XWVb~LEiy$@Y6A@o5O)8lf3CR z2%Zz*n+6xQZBG*mN#m;{T`cK#81adA5?9=MsES1~&-EO}X- zSIeU6#^ECC#$}c{qIDvVq9YJzmER~PpugL`Dg6ZbqISoac&FCg}z9@^2R6WHr!;;^Qsq78ez9W zvlmJkwYC{PtQ5IaWX|UaVP*?>srW#EhMk`nWl8U3?~s+G_`%|)an8kb_(_?0-f0Rb-WaH89IE(RD$X;E&CUuhbT1C)Ff^Q?a?Lx2a|Knphe&qM#hRa#l4B5c zfZA0@xN$TXE73Fbj*W*%?NO%M;NTMKrA(?AjQ;`;&kV| z)S_+$7T&iQ?9kvh9o5Znx533gn?jgy(=i${|A-p0T&WX?zvRlAdOui~2X7blvZT(+ zAZ)2SS<)yqQ{$zuf4g918#iHV>)4%s5$HBT7qFep^ay5F)R?$oz|kb#E9##VPGD_R5XB^P?-l_q_i4$2&;(gpe>+`0_F-HGvGef28$ck= zElbz0woNvpp)xg6#)0dGu5%f(Kn>}eB|My* zt=Rng_713rW&ZKZ1we40^MLlcs-g(+!tfCHyj;hAJ-fe0TJI?rOR02hhkV)8HFArn zHP6Fz8v}V^PL1HcQNEv?OhBPb{1x7|Zw7;=c4ExP<@<2Jv7mBlq(%Do zfI{#uX}qk@Uxlb9^MqA&5_9HPlR8I-=C5$6>ka{}O$uELsPBQgW+bsnRqa6U-XMf= z$&?mtCBxdNjYP@VG8ajM7uhY_S`Gmdi-$Gxn}7W;-*E$TyAJ|@9gVWNm_}9n_+Cc# zo5y=x6TqR|o8k}cE1lekdS9Lh$%mL;6~A>nrITHy@o^(f)Ic1{P(+tatV~tNg;vs` zrc?qcK=f-HA{XB0z3m_RU#V^oL+-kz%qz5g2(=bBkUqd>2ZYz1u_*)eDMP6jNybou};< zvU4PPk+8T5vRs%F*~d7`~6lrcyxSofvfdtn0A8dOkRabOEn=HD=NB{t_L2K9*)UcLWQig%Xd_` z14mBfa#e?${67#$I&r9~2ck4qoeW@~(d*k`%FJ5jhY>4Zz+eiP0czaqZaxnvv2m{d zuF=4--}hfD&{OETy|g%zW*R3(!f{}-hEZ&IWK@ zhFg40+#@9Vb*$vSUwdZX=1>3RNKbRj0eSh8&sKs9L&|=Qc2U`XgoFmz~ zHa`rwDH0W$h#CJe4hm?|9hTs47SmY%P}Y%TVjilx-;2_qnsS|XI{2ml#_Z`SU2E9 zvl00aZVb?JnzhptUay$~A3xO}>QC`?1PF~7Du<)pm&}?0_W>%0uVF4S8Hb)TVsxdU zf!E3`AVa*PqfgPicsuY>BVdmY8wh`+V4N6qT;)hTGB5SAp{omV{ci8`VE)sLtRUj80C}n?m~l-8YuXdl z90{tzBGwf#o+dp0jD!nItUE2-U|CC|+&M2Bo(8Zgbj0Fd-zRHIJm3efJ~;$;U`n8+ znW&F(IceQ_+Vz4?Lvl?haO&qdR~$)08uy?M;9JD%aq$m7KsI!+Rh7U;174x_lWQD} zCtz>a<0JOQ1EV$9DNu74OsYx`Y`Jyz8$-KF#@NC?+wM&gHRg7(D7U{dd%$*d?s7T; zmW*brM4ys%F9_G_)72xnR)m}1#hrj%bNE)8-s1UOsL1w#RGXrOP6Oy)f5TK@XOe%!1JC(Fx$b876C3m?<+Yt zwXWqBg`R3$nS9}wxXe8ccAuni&i|Hr7oL%ROfdkwJl3u(NN)&YJWdSNFJ^j#v%;C3 z19;Sr<;~az;HQA}vm7@uGHI>gM2d&et_skV>^lsAYRjnYjEw9ZB<- zPs{8{=)%`_uqq1b$JK0nExI7Rv((QY8vUd(*czoBfE~P~_O!CE!aJhZNE4TDP9@bs zpe=#Cj$pAsqhHv;Wo?mEef%3=o}4+W=O0(qlI3v9!3`qRcw&{sh1pn{E4qvxFeg7( z>?0Y<;g{>ePfRDYCmz!PHz8pOB+8;PdiN`@o20RIAROH)tx99=ufkBeUiA;3!#ry! zVOfcWELkiwc|N|xYxb3DEOO}ygL}>C(R#iB?R$);v*{tLw$8rFH+%LylZMzVuI0Ne zvt;*==i)kP@*#GL+qUAfv;TA4leim3SNG(u^&M+CL@*=%D=zWuqsHIWq!e3s^e$z- z$u(7!<(pmJDKpDMN!m;XDDfLe-6|*%$13IVjVcFhfUESF}1$Yyr;B4PwnpL8B`tvlBtaAw@*- znEX-agGBcymhfqTK8RD9q)31gv_I(F-A`hX3NR-y5lK$;^LdY_2}Cp5#64wGf{&+d zQ#!^RWWbz*yU+k@l^c@ z47ch=60H-#;T%gKzayw;-@<2!Luk3ZvX8V7V!pQbRRPBA4Cp1`i-a8467rQVWWA%6 zj}$&8NM;s>h#_HLDC94Xw>_-G*kVYip@hoV@KBr5*fY`c>^NY%Oh&0738WU8WHwW`%`e*{ zC>7H%oULiFn{1D0&aHc%)@^uGAE`RFCUtg&L!`DlNoCXha)-*fZA{lMRk*XrH?7?d zKyVCXErUtg7Ia&=DWX&(+PO$lTA|nf)>=<|E&mRRIv4lVb3eMk1)Z z7GWjHNttO4hISl&u|Tgi)8~{GCP|$TW96GVQxT$6I%swjKCbLQ7>Bg09+qpsP=2JZ4EhP?wYoaA(4Vi@rYfBLc<9fX zM*I`J!6=PLB6=H7#r|m*lA|!PJZ$ys|Ki=t>xL@&pB$pP^Ou`adjK$-uQU774^D?~ zw!^Y72PT@KZL~A>B~XghLo(;8Fl1+XpLk?v*cZG`U;3fvP%YT_Q|{5Vv>^#Q^7##? zF4Kt#aipB5EE$*5Yd0D*G}vT<6E_ieP_rtP&lWjhjFX8wlrid#34jo?U!zc;9}49t8&xgQG!qUbw{ zT!x{B6sJKibusWqIy*eNB#af!xPj}ia(!oY&GO!;sW`E_zdIctKH-wK8g3PccZ*BP z1Uh~I7X*{d#eRaMG<{o)UZep^BZeJSmh4iIQSlT`02F8>wICsb1GCj($#$kejXW|L zLt**aTJPTrF9{x%o2UOY86`;Vm@SExLRSnt65=-|a!83f`=Ud(yj1q2T3SW;Q`J+@ zc!ZI0jgr+6ML{|1^wxZuNudWp;tgBUU)g5RPsr#WwK|Fld3Gy;nRe05eS)KFUzxve zSf%K|K0)D6I!u*4_bO6TWYn!3863PtW6x$DiPsK9n92-zYW zap@V5sy-J|1F_DR19)_9P-mqZE>?<9nI@@}tmB7DIi>)A_?BDpkuz(BF3U=Ll`=HA z=EaQ& zsFXbda_68Y(PSlP-;d&bkMR@OCO<&d+=t_4ngv};b~Jf2V9{6!(T~wlE`&iC{cSx~ za1X8;-Lwj6y^^CDnTEz7&SdH3sa+-cx2xO%Y!Xx_N@PX#$B0;TUTx?62fxF(csOeI z!qr*X>@8`t<{`-Xx_n*ye2mrYE+lPEzJ^LrceIznWWOdaaFkd^%2$5pKG+^g*G{?l zwWb2zd_}$P%{ejr$=={(g^x~Gr>7lqAy`)~uo7V~)`Ot}T@u6(MN2q*&PN)wTCI1RXxIk-cMKvMxrMClOkjJk=fLTz@ zZ(o9pMK$G0pUMJCp)Et_GA~ud4zl-hKBihEDeai+bf49d0cfOC)DQR9ho7k&vZkQd zJSj*K=NoD!G{EGo`yDDNaia@K-ZQKx<^gVoo|eH{q-<7Rg6LdGpPFBTV3kuc#rmyi zA_KJRvKTCP6EQ;i!a`=_Qj-ACYkXAFCj*T536vVs0p>=(GDW)V?qr<|9M$&f_ zWhr)ELI(f6p=qKfI>U)Xp1;U0NJDbo2Dm)oE@0F${C6vz)0qkuE3SZVsEJ`EpsbBO z7%Q8o_ibW+lKv(#^N&f0Lw)xWSQF)KC+B0Sg&YYWx;8ZGxC}ToiNFoOHx@l2N)*}O0jjE7-l?|WRp!mJ$4MkS&`dtcX@qI~u ze~*vL?esSn*T*3X*l@+O-xxNwBXFo0c)u%D9s&Jt=@H+!f~kZ+bgABoX6k3h(W3TF zN}mZgcQ;hfmZwgbCsS;R|B(tl^6oCN8=m4TrRH;On1Ec*i2fA|FXq`&mq1WV_*-cYtNt?ccZ z|Kk&csVwo8{YNJ-VopjUAeEO&zN`#YLc&ecl}YFfZ9950Q`9{g_!PS%Q8!c8QwWaj zfFA&&&0)>OPS^;>n{$eN6eJFsu~%reOWd5V&co_ARdgeyb56X-X8Q1rk=y_W?L|w8 zMhzQFMUq2V)u2MzvQ}Kg5Q|o5FOC<@(>KDFW(;PYx9mpdewF#*4-C09V+TO8*FDqP zzmH~QuAgZiO*Ogp7g+iH0M{h-K9^^wecU+Nzs^oZ(0bn7%%Kc$eVjg)K5tU?_X`Sg zx;?%wCl?3l9$qeGE7dmBa<+bb3ekFpf!KR~EIJeXWU#DJIWP%iC`m7H)74?$l0zJh z&)^0qhu0U*#wo1E>;#rEB9wh9s>3#oY77=jYN4~KVBacAzaHRkD3th>4&7a!v_oX` z8R=_GUr-j{EDEH8&MLCcZ~M_31x^ZsV&`X?s1rUhh(QDX{3a#UbQw}zD7~`wNA2nX zm_xp2%e!J;bMF6ebx+}uMO~w)W81cEqmy*fv2Ckj+vwP~ZCf4Nww-kBlm5Q{oU`|T zQ)@2Gx~ZyXt{USVad>gzenu-++6^M5@j7Ut(mD>rB z!&(d1Zp+3I^jmZcgX7oM4a2u1sI=l6b)FUJ^Bm&}&ee!QmXh*&)wxIpi|W7D8* z;4Lm@cOZ4fV0G$Ss%8ObVm-0nZl(RrfL}p8>|QiC&%$75WU!l7JBIWIF^MW!bleNAS@cG9?ldWL`8EsmP%~zkO>-c7p8WbBKSv!>Q#!$LgnxbeeA|U zSnp_Tjf9DCIF2I2!vtD%My+r=1&j*7@N@v%se{H&On{k&t6>l}b{FXh^(bq%#wt}S znhtH{J$yUGe>A~|{*?K$g>_N1?5q^T!O($pb}HIvf|!`GpnEmsG8WM9sac~8^9pIi zTTK==%ui)0&aoqtCn)mkU)3?difJn=02)Ms|B3;P4Ha%VX@R(M766{^8Vr9ma>_Rt zs60cRJ=su!iT*>vvctuc!6GB}SOhrg#Nt>PiB%c<*|C*Ay zo%fx319yWtv)+T;k?Zp^EfGeE`eEY!YX3m_92~GGFCkpny0{lj*0fe2-$_x| z*7LuSfVQ9kEQE)Bpt6-BTYjZYQ{{of8$5_D`_DUh;8qxMKdDPwkiv{Qgh6b0OoSN~33HugXq`Xam{ZCiMhRspLY8~o$s z=5iOBgoazJhMQNpHcQFL=W?7Ma2V&?Ev)HwwEa)83!w+*85{x>#&tQd z-~H)p-cp^Nkk*R>?QLzZ$)#U8U9R@g8_kbFU+9TfOO&GuPkejE2`}}WMH7+uNymA% zWb2;G`}lbhPMjnt`cU$Uo?TyWYFv;+eKzqxMalfKBU;Y;3JH+cfp_m&^v#A;ZB(H0 z0FAc9!yo>pUPCu2P0D@QSC6VRQPd-D6|e;nma7gFzZ(37iw`cmWIv@;{dUALSBs|r zD$)s{^BY236x#^m=X=+Y#j)IUiKcAo(Ykp4PDzW z0^JEO4iDxniw&S;+ZU`$R1QYts03w538LWuRTKYVxQ?Iox1JAZU0QNyncV>>n%E{Z z?)E+L5Bb~k^bOmSiq)!;?S57e*@ZHo2WR_EA$yS76(i{EC%Exdh>_)~WVz;z7gu(8 z5~uyT*@7aPgSv*JoKx=7GgH;9kS2bcYmpum;#IRECn`22;PRPsn^?l^z)=kxy`pvV zVd8!##`k&P`ip7UhDTC4wqf42B!{8f6M9sEx#D$ag$BwtdJn<+oYLzkP+X&JfTpL8 z*-mWB!b~&Gy6gA@Z(P~5R_Kx3wLG#;_eAljk#fd0hDvZNnx?W0X!e-!B+b@~y5OtNz?TM=$%}A~Ml1a;Ts`E8OMhWKnC?F%;_$*67CR33YDO?fuiyG z9H_5DCwe;DyiTpFALV+5IJ31nat{&Ej&Ga^6m%l}v2POTC+Ng+I9g@%LRI{^A^Wr8FA69Xa}v{J?@DD!JVV@0B#G zVc6W`uxic3Bx@URVbJe>r(jVn1eigF&n}Y9@2CS#QsqnPQ30FwYP;v!#i5~-_gK;0 zG!`Ji!tRjb<{c$TGi9CV#(k>VR4sJBLw}z`BA`@TLKW_c+~k21={xeGtY*6LB2tnL z!7syBnt_|(pJnY7s9nnr5Xtz4kHl3eC)Y>XFbUg@f~&JgI=_!FP&Q=Vtk>$-n|4XT zZ9$0TmCt}~G$h27yHgM@9u%Z|DJR5O{#B3>D6ca6&n9d3dp!E`*>$dA@f}hdAl}-<~iK){zuwaYh;xpQ|;ZK*Pw&?q}#xQBX}No6fOG7ZrK@>*B{2&ck_sY zH+_k3$nQ@d-KW|i7hjG>#q8|cx8pty_%POr`rpTSdcIt~r?>Jnc(8|iVx~->XnHIu zw*pv}k*Ukg0BqoL>pTsi8svY}pigFtobM$-!SBZfNt{A>F8-fVXztp@Mu6x_0*6jt z6qHKqh`CZAyy^XQud;5nhQmkSYfW^b8lT(XL%Ze#YMLN(2He*+NkRlAw@P}Fm}M;= zAQ_8ByYk501F!o>ZDga~$e0u#1&@JUgDcdUYgbm!OX)7#*0ZRYOblQ)N;Y5UF|XKV5GqztTm-o{^po1-;me{|6cb{h!dF{J+qE3wDd@eA8Tz2s$Vz z{*uHzP_+X&fYLp6@JeW@ovUYCJ%{l1E?|J*{QB+UfRl<-u0f5iAS0r79snz8Gu##I zVZqDeCKGBaYR1@T!*Ax)h)L5bS({Rq#4kZsey=MXVthTu9Ksi)6WKGaMUs*Z?eHg9 zE5=AJ)rJmXAY;MC;_k4aBzmR>zyJgqs`ysduWpFQI-pK6P6Nm=oZ&;Y4Rv4op(+sg zxzLdlp19%^RW(Xj_%hBQomkLJFSZ!H8Y_DI_po-ameFY19Gtn|LgPGq_uMUU;IE!~ zXPBMzCvG@rPbSe$(UzCaTu|2bIflcp0_aBzh60ZQ*ddTQk|X5>YWY~G#>#rDz*jR` zcX&sZoupk*g=6F<;keE%_IxP?$xdPj>NT8g-AxoE6*ScJ8C8meJz7@skb=+)G2Myp zjy<`Zi6;G+t09#Zs{2Dq4)}A&buZj)`Z$v*Zoy`XmHNGCxJItvQgEAOM#{>ftvN^C zYa``OADPP#bH%!J)3PMv?Qk}|L;ATB53gIht_Y!_Zr=8>6=MTCjFRpHN{!v^56#!D^ej@n~taq?oF5!>l6g)W;3S zPI;Z?Q6>hAidJtOI?WS25ko&;4uJ2p0x8-St)o?TFkxU z6F#&8FR3dZgZ2g{6rVC%`u0e*y)eDXP1r%T8jHGAc8RP6H-4nPG8|Z8GBVPV zhf;6!8xv(_6qx*pim-7tckKQ6I*?S6=68R*mz(}#B7zi@L{8-?h1)pRZRB#k3qJ=S z%mFpXYge-P$~|hS8p8ImZQ>|g-5Q(J_$Pf`H>&d&N3GJCt9u`0=bS-ioU>?Ban2msVN(1Lh`&)X0*McKzy1 z0R(n~SH6zHAK6t&yjqXbugUcUY~h7h(T};DFlke_9-R16xXs2s(DD~;AJ(ys)`3Cz zX0qZamGLdL!CI1VVA@vN;xhVaByzF}lJuz9RL+JqOPE5uI_rIA76YxY8y+rs-QLbV zhQEc-LnBfQ^uifo2(`QU5S7JaaJ;3%x|cp?d?f@sCC8EvN5nbhiw@JRJs4|SuO`n# zqiwKwGv;kNevF!r{Z|@Twu9!6M(3*5Yj4$}6|wya3u}zs5XbcSmIjNvZV%K9-r-U? zdvhhN_y5=M!!ZJozBl*g$u(FcXMf5&jg}%XScJ&0r2UGPVw3ZqYp;FH!lejFUNhiZ zQupRu|9f!Ty}0$>|G0&6OWOWa=fAgoPw=1nzGwIz;$OpUSduHfQ@Sv)X8*GW0xR*_ zV?L5F&&^B!ticn^*vF3ZJmQoyc)yid2-#-b=C?J_LEMV^wg&h6@8=5U8n*7$d*wNC zRnEDharVVW2mStV`ypzK;)=?YWM*x2oWR69YapOw%Qsi2SnrVUQ_ia+)uM6MH?p?h z)?j#gQ&oD`X48SkVe_vwk+S zn&bLs4X*xK14Jn>H_f9fRx+=ET`4gf2xk31k*H3)%fhilY_v;(hDdNbi()h1&E(tU z8l)+9^Zl%aMw%k5(-02ai%v<5933FyMt+8WQ-c8i|4@T9gdy0Ut-p9{1XC>|y-fS9 z?5ME}gi~-EXpULBm|*8Ucz4R9p4^9CcwO;HW%zt(Hwv%ZgL0x_p#-m2oUa&oQ_6Pt zo1@8@AesEHMe?#gde@SUsw_c&(Aeq*0oxQwaHDk zXPmW~x~ZYu1KeGpGGRycC^&(;yTXW>aIXKbL5+h>!b+ftk!`W}EADCuN3^|JiDuSk zH}9go;|$&zM=J@-@!E;3SmF&syn)hMa<#!UMxmR0lxWs@c~Mk2i`BH97@wAZ#<3`b zXxwh(gtj?ZsoR|gSRffHTE(hPzq%#w%L~CG7Oa>$^yZfI7NzQ(sw^tx%4&}O zsOXrjo966SMCMZB#UCbN|J(} zdmX>kS$N!2|L=h;#-}+5l4>vy50D&NTgaQlMg%1{%gt5*Z2RUz({4hcR~y>`@rTR^ zOUa5u56kBLh$*o_Z%B?Y3>Fro*od6DI}+(pj;RNIMG||ZbyfD<4lfg^+LY3wPf_-$pf8Q;{Dkg;2Gfy>O^R!BLauhp+JEw6QYTBA#eL8MW z+q2`2)ehkTBa}^p?8o3xu(2O%P-WR)5}|sT21L)OSfR3Cb%iu-Tb+r*4IR3aJ{jkF zje?f9-oIi;Kpba7ykdEifH2`$_9mb~goK%+q(HRMmG`TLJS})J@jP(9G;VRzne+H1 zk9SUP!}uGm;)>7O+h$iQV=Wy01?`nUeGOe-g^F2PyQidl23VU7YgB|;V~O^3L`(S5 z^i?F&s2$6&^DNZmU9Z$@X{)F(p>!0v`eD4g#e8VZEmV77Y#~oUp0vc%zOli|RkI$) zqS}aDz@`riJ$3!W2@Rf=&7|hK$rMhC!**fNdzcs)C0eT2zWfv7C+yz~tVa%&!eT+I zL@yt>U!RAkFC`IQ?axlHXP2wIe`PI?B(0a`#1z+ELybx97NY&zlpSlegd)4e536vT zOo%9%Vf!Is;ZQW?snW3IZ(IyFKM;2fCFdzEs9`T^VSR3-;{pr3npFOS4b~XfqpBon zhzOikESGiVv_kC()T0f|xaSAPnGXUZR0`(p-PcMi#^#pZgS5q-#KYZBM?4Y49X>zz zE_|ac(pF^J1Ott|iv3s93XUEE2N_aLA9p3@9wt!;Q{VDXtif7BAVy8bjt=-}pejC! z81+*kQ>TNEa_imxiV+z@5J75+{}i~>oRfaAOT!uu>bklbH`#ORZsXk^ll=4Ra7iyV z5xT7Ez+4kFu~?A;tV-U$&G8hCeyfs+V64uodLt;iLlqR0Ol~Qp{>=0xN@uK5n z+X8$Mu+ZmFUab-CTMMIm2?0jF?R?AP^qx74_=g}86H+jL765rwh_!oS&_RB3Typ9e zXDuLJ)>D4VmhU?CL=|V#W*{V;|HBia8Q-xqq@>3GZz(I%gz^Jni=GvT$Z%EVpR(8X z#Kj%>MpAum@5iwVR8s!{VKS#S;Db#3-xIO6=-`L8lhCU?s4|&&?oymVrjkP=g-eqi zRpA;R)9;Edl6LW0QD#slaegxBSACNH^rp~fXw^%civcF45bnus{bzJo zD-(|GhCyc@p_US<@+uaXV=c_93_hNZmazv)4#ux;aGtPdb%3zPG;1B((-BAo@ZDM@ z4U&CsVqp{&azKw?Co9}qW4V0o;!;}#pH?7QOBFxql zn#L`c~ z-Td}}gG77N*ofsDYKaZFzA-~G%d<-CxIQkksxF5)(kT$%JS@{85U=(^lB5?|$|#}< zO`_F14ASsbj|89_H?iEv&03&~V-CgEm;U>HdiGc%Y@OPPp`V6!PAfl|EXTjQmnW>Y z(!C?H7^TMSL70epsTu`f4o8lbhFvoIqkM2jmBDOu@fxwDID1Ol8Tkkpg|jG9CzG)- zPDxoPPb#?jeKgwV65IS4*)iJ5vKDTF_k)%A+t|@B+NRcO>+S_x|n<9o<;%=`Vbk=>>W7K#<@jBmko*N025;_p&65aJfJvx-m*9{|_6i){1h7 zFE~5A^?K3j>6Z;gWA*jO$cZ^V zkX|Sw9kmzV4(Lw-S?NoDB-csn(D(bEpG;=Kxwd*=Q_mYqLVDQ|i>GMevXA zDm=&CpOJ#+60(Q|#NRFA1j`5m+lO%-n^9PN(e*+nleZO_*x2x;SN6S0f=g7OSPGbn zedm>1lt;-=F=>_g_K28Ev8b%J>Di7bo0yf?qh`*Qjmd6nDJ5l-_DMzP8esC-aqI?mugga4U(i=)hh33r-|R)inSe z2d&nc5M`dyPi(H|pEVHqwgw={+M{Sd0I0vOA8HS>r_1m7A)`C*2;U7ebG9OUmi4E} zT**yW+GJVELWptgNaESi1Go<(08pG-OccNAQk_c`@>|tSBITaWbuw?+K;WbII? zEx+=eM$My9b?P%6tO8~V6G4fq-IFYI7ot!HiNx`*M?Ch3P>P^w`TO)jk|z-4!mYJu zQT^JdV7vN8xL|`dXk_9e8-U;V9+B$qb3brC)K`#cd5z-&$aKv`A$B3t5%lND4#zBi zodwBXrV`S1X1J=qezu}^mez7Umv2XgX0L#qEQVHnnVUGN5C$cvg1M42DkQbew%s;S_NXF}LAlf+v+f|}U zCrmUXzuzuLx4{FI9zc*!NfIx$DHIpi+n1^j_(4Of@B4B;?MoIB)u!V~jgzM!CDY)6 z!b+iP8s)bLoZhSxVY9_)i*~=QsY72Ig++g0$b# zK#gr2gRQMQ(e^;H3$tciG5NCFY{BlqnY=nQ_iu)ciGxX3GlOy)K$!V>MVBPdtCio} zWf<348OiiDf+lv`Mi8|bcSZXAu0-z>ItDt9{AFJ0_b(O7kyyWybXe9B4b$&5!Z;Nv z_2=p+?`!X>{bsvHad-C@Ju#b$<8aM7EB63wl%`@AsFaz!Lz~$6dRST4>$gKOQl@^VFF3{Jz$`bnO0VB2BkM-Jk21D zOQgBIy_*7!n^chx9oFK+J!g4u^In3O6;lHOUbtoYv@zXyji5$H|`A( zTIXqcn_vx4sZ_jfu8nc`p}yq$+zNdi|0-r%-?NehoLxj8$U9g4-r%CXIS3#iBI+6D zV~RPI2k0dlqJv}`D0&m}Ic`lTOUKFeD%qZdfW;;>Q04VSjy|MuaCZ*h$NOdds+kP> zrPpfHDmRIyMI{h}hDyNxAtVxNDwatwiY&eOe-flJMB7ZkxlA0A2kHgMxeJ7miR-wj`n{K$l$)A;b;mMK*(=aqroV)2 zvZs6AukT9m+k^)Qrin{%obbyPAxtX(_32?CEam(@_gGFiRCQA&`R;Lb+{}N0`!zuS ze7F3;E!dOD8@YQ)@l6#RR913XCodATgNVGigNUqt%7O*dTOWZG40J-Drb2RxK4r!- zgvW4*B7N2#a|^dP^Go3G-#*SB3rOfk0*QC_F8C?V<}Q2c3~NJ5mCFW>6w+Zd&a;Wm zH-hEjPQc%d_vd60(mcIgafKeNN>_LHTVN3kEhGF#{>1~zrjo-%Cn39hh z%QYv_v$U=2P?Sf55V1Pcj>DQ2e~o*k)J%;CD*qzd?L|)>UCgl2KFD&%V!h#qXElki ze45J=I{7u6V{6gI`;4u5IV>GcuV#~{ue<8kzIBtjB;EBl{6~BJ@o#~>ix@Rg{_kvy zqG1S$btWABdU=?IVpO`+Dv0H&++UEvAI*W~zp9MeX+-z`l0j0|3R8rTFO+KW9-hYN>=2?|t!K90^d2*zecNU|>G&BH_P zt$=JZju5)89wA+iw>|foujPo(VRULQ#IHcdX-Z-eiNc3Yagdz`7cogs(L!6UL2fFj z;uvV8$jjFw+a$!7vxA+F$2(!zY(hVka3-3GwE+Ora-#~wNNYh8tJ&g%1s4I}TH^NGj$to^= z_v?=V&SB<0Keu!Ltk%VZJch?ocj#kNiFDu&3U$q1c+G6_l)5?n#iPDB=uBY1xz{en zGN_1Wm#V0gn#P_H(>6cdE{Z+GjWyr1^|?-`N{e5GSHkGo*ucP+Yb1$)r%uatZ*d4l zFo-RvBnbVh8m=y*84v2BHlo;YKd%?dR-609_d}vQp@J)Fn1C1e+0Y+%!rS!Gn*n^e zURj_Zz-!W!q}>J0>Xe=iO5+s`-|CIay{HKxh0jcKzO!{(^Cl^_dM>?M0$wK7{T4W3 zNyq_!7sF`}UZtGk(9J8ikW-L|p0?q489^%r#SpJ3-ir=DkbBE3u8CxJ*lBmFz%oJNn|v7-!q7KH*>C0Y`KK?|Qh} z*Lr2Y>w+%aQ!6;hVqSDEKs{*!siw(8*v{2l9k%@ha?GD0_(eMO`~tl9qyL?a(u~Ke zAX8PdJK&V->&oOrVD-g&rOl;ZQzyZ28TZ7sWKqZVhX9$YRA3~W()|`4afZ_Q=lg9XU ziKGt#zG1Q=s!5aE0Fo;SNl~fO`V~*4IWg=GVf_aDnt+v+`PXSiT+sxJJFL4-`k{CB zalC&R#Jm~Kg(R(ip>!vaUTh3x6w2EzP=~kSK$qtgl;G3sIkD^kp7~sUYKtR#t9*KR zkoN^hjAfI@7l#AQNqAAiAVog9?+)uvprpg$K!LSl5z?|p)CScakM0gQh)L~>blAA+ z^ZQN1oA3&9vrCxR)!JqIq#`qBXWHFMQa&k~|?> z-dKLC07*o~)ZqjBo-xCtrP07HLQmBKn(H{yGckTztm@ehnd;*VR?(mWy2nn3@`4%R zS{5tO^s|}gNDI|R;^BipQlwKdH<^UT!I#PDIoX{!Ho$Oq%tQ9y8w?L2WMD|}Lo$ck z`I9aBtW(Al8MXQ!IQDE@71fK>xwTaIOyoL6FQQQqTeTYqbkg{)UVDnbSJHR98#dfh zdegPGRdAErN;EX8OLqjBJ)zB;;X~H$!!VXeKnzYFWg+NC z&j`~7##&x;!bk2RG^Mm7Jn4;mCuUVC3UfsCpg*U_VDrJ%YXc~(WT(y~kGqbitn_2w zCScn4OJt!Y?=@8`*cDjJCK2r@fQ_qLEQ@56=j#25wOQ*8`}foW=O`k5biVtV^<&h# zX zSdI~x$FsDdTpw|nr+;9t0WMC=aolbzpV zobz*pZr}Vxt-wU0)M>l#nnsMtsWLIiMFG#(Rj!B#g0t1qu-M(5QHA(Eg;cK=R z)S0mMK7Gzx4KizMO0OqTyd?o|E9mA>3z$_h0*3)4(Mq-7{z2fJXs{D4^ayVwF$)-( zn-sc~QBLkelmy;jRDVJZ$m4(Isw#%}J$f$=Dd}8J9E<(uJA`r23Snt0U^4iXq>2sF1{X{}1))A!nf)k~=R%}y-|A~ee9v7lQUJ4}J3lh* zh}QZQV=!GC_Q^<1#kd2;EmIc}c9o85I;st9bzO7Pi z@LH{!2Wfn$i=AxwedhzN&!#%QhV!nHq5>_rl2ulL`lLjS*;MNZ?m5T-a(W1{X$i8lRr|=ZVLDlXuMD_C^x>jVR`Ias@~*<8ap;S$cbvuAAf`kjiUCG2 z*H;Zj3ukm2dKkj4$W2XEdoF!rpgX14`1nKo*zCHo!?1xskZx<9J*g(>JbBFSn38bq z@J7NO0B3yf3mmbq;5~LIgGxu=Ug2|iPgw6X^oQpIWp)t28&}hiwTcw`^T?er-XFYt z@N$mn3f6YNv;qgqr;^@Ak(0vN=-k7ff_@i9 zwe(<2eQe}V#UlS5vraS zd9+lfB9bG5_FH^(tW?9#Olz60A#O^Cl)Kd=B2Z_#w-+cPUaB-DL!szZY`>=5JBRED zR5)pmb~pIDZ3QPzZHafAzL7UVsFtNw4I)T9IF3n^(U9T7a_&=J%x(MJ-R4&4rgQqe zzXW0GJ@8lBz5kZk9OOtLV&vs^A)@qa@8>=6wu8XZo!|FoItu|YWsN5bfth4oy=1^I zB5#|JdF*t?h1m(ID`h_K((uFqE7FmyD#o72sTHgJjc4ezcHi)1 zj%}C+e@`1uCoAVW7h5MQF;Zf5#hZe>;3+K;j)fZA=#7&aJ0ZSKj(i~8w#IsF3O5uA zLy;TLu$;1G$wJU=d-W$uJPWb2USb5IpuW=US^K3`1_{*foS=mqB1g7M1iZ34&2#jp z_;2-6*ZF#Kw%<1#SYaeD3PJaE*;%7gX1jw$d^HR1&@X?~g@|mNc3hk-R>)RL>ttPo z-z$okG^%7wCy;j=EKH8g3%!2Z^K zz|*MrQRY0y>LY`s)WS|`gr{9)D!_){PNg338hWSPc%&_5)p~bNI+a1+wO`e25Lmtb z(Lk7NVo|hQe;^1?qSi9~R`vz000&s+Nm*+l+)@`G#-Y!9g4hCf@%kKoQF?BdIAo?= zk{I(NA8{0ekhIjJ?rwkidlsZkU%D4|TKN*n2I}qOYLc;KYw2?VjYpeen&=cY~{}+U*w8YeWwE-!5Le4%z(e5AY;Zf-!%ugyDm(@jl`ybrsd-m z>H5iD32fzWU}^DyzcPc7#Qy`M514KwugGx{B;fkDVV}Xw&6KhLFB7b5rK`iFunPDg zJgqI)_i~`l-mFEt#%zfUqHXC6`Y0Ft`1tQf#S`O*MLG#VYa3OvN=Qp~I;!T)^^CTI zaCMuwocQ2;&sEh*y%TxU(8MnIhA^(9=(5S}g#wT|3wQ=)%m)65cIddqpMsQO6?}qv z=ARh&1eE#gF|KPpd#gJ~)-tbZ&Al3<<}hbm6D{0flHbCR z=wa1>C%DZzhogw>WZRy!F0*CR#P4T3UL@Hc-L;c3_4;nZ2J3a?UP zj$VdyI+E>RpoTpvWP7lW|NgfY<|sB()Tg4MaS4{#TSqceRK?#a!^4SS3HWNiBGFcI zj+oW@+7}27?Ml4hv`4c*UR$&G6d8I20@rcTM6$5918rMEkNLj$LDr|IqnQ>}Y$)|X z=tI35nqXtp5*ANRun6@Vp>#E^j%Q0h0MFHPiEQudk^91?qnnAKr}N=EQgn2J;Vu$5 zdDT4FS!a)fDBg_8u8oeb0>}#S{P(w;aNSU+d`Gs5##I>N)RPiaU&;n}~ zXbdu=-#DLMlPA5kjzz<2TdqaQg{*D26alzVO?ot2DfI2RlAB{Jx>@|44mofz4B7f9 zOyUjgh&qqQc?GQ4xs;&y14p(jKc1Q">3>sT5AEzePJ#QrD)V?~{&al~+XUUJ0iPvEmMLY6;~Fhbjhx^zpF|$S|TS{C@^UX z6Q`3IHD+44lpG)6MMp-JtfHVE>)k6tV#Y^AwD4bk_Zm0W5RwUtH;F8-W)b~lL%6WD z#R=UAzbpgG$=54hZO4pD%_7j-*m`rjd`Hk$)w%Pz)(Ds*ytqzDIi4;sJ@ItiDf&P& z%*jSvj>qR|0Ej86K5pq6UwtR8R~)ud9P+KoQevgrc2ZbUo#beCY5U;t6@00 z$$kz&brmsS7gN}N0)v+WDR-t66AG20k0+DmJ#uAq3^x{~wf4l^y%0iFJ_}JC%1h{46_aQ2{*3{WHNK_e&pl5r zm%b(BReVy0ILp^T%EVb3i#+?4S1|*3EN!m31-E^bHJ~}JRUcGE=hS?PyLmG+REcPT z9UqE(1kFu$7P`@VU5^j!1cyY2CnkW8IKF zUe%96Pw;m-T;(Fj?y|D};|bWX=FS8aCSsPPe=l)h6Op`P5{Bf5IE7jTaa$}b$vE1o z*H{V_38*ZOlk{fJIUlTJmOuCpT;+}TfaM4GUPYIl)~T=V;npd9-V8#D`ktF9C^OQXW7pHJwWus%NyWC#ghuP10@VxQk|PYP zHiw4%l_JqZx>#;8>Gc11$;*wYSys=-%~0i9{HZa%@>3P#$OfoL12Xn6OlDGp`{*BV z+RE?imETl5{YQl2mNkTGVH3_Z{S*1|jY`iidM+XTG`!vD=dbjqy39h7c9bRXS?kzK zot_X*zCRzexjY(rOmW?YY?H0O5yfBDA5$2f*-BjbY8(JDjW17u>o{JCEr}v1Px+b* z8ZSHABS9E!*a^fBb3QvEvuL6&P#~IXQ}77I`bG2=o%&Lw+Z!zTZ1n6BQW|KgHxmP( z2f_`DKa9A$-f(5A4;{EkU?;rmyVS|&4?_y+3Ie}>f1}qnAxnkue62gH1Xr(8)iTnL zYoP)gV)k=eYr@d{6kG=+?Qk3F6F-K7Bhy}BH&Bl&JF)hAJv%;3uI)nMf4KbW|Ne$S zKAB#`P88!e89$4Dh4(Sb4;%ptm^lhNYCw<7GuI2`#!@1`F>z5bh~>n-qH@yp@)c^F z2gTNC^m?R?S;lGP%DW7V!TWsOGz<=|I6LSqoc|d#YnEkVzaE4vU;S=a?pDwXsh&{Y z>p1lcA5i4|@;)7MXxV!&!Q4tgWwp1+J5aIA9gUn4McdCOuU>Dg9Qv2KR9s?84IuL! zlA%4ikzl$%%KOf}`w>*ocEFvWa_!Yzh4{>Hp-$^iLGQ{lHHoB-@Z!vW)<WddApMLh!t1xfRMr@+I4WR0f# z3*Ban7NGrOO38Hr0t*-iAnDe_5TmbGGzUfmHIUY}lSc4Zjd*FoT6JA|#CdV8ZPA9K zI2{N}>W8GUocR4{n@F}Sy!D8fMw1VtN%J)jozx{*KQJFUiKMsbh0RHhd90$P^uOScx4(jy zfi;d&-_H8I>%8?eFi&)ADkGK|1v|U%ZK_Y#(yOaZ*VYJTZ9mswNbAikg9Z-%P6kKe zY#^xM+O7Wtr<`{i_R9k=M{(5eSre83<$H{pE##>FYtqXb%k~EXepcziekaW^resIp z`rl?{Kg%qfp8)UdPj+(-_2Y`Kk5MNz!((c6|C>OG$7QBIqnwb3V|1s>F4Dj%NdC5) z4<_bG|Cn3osnnmr`$G9jBPh{P`AnIj-5IPV_H|$MAPM~}u%%P6R4`Q*=*o_1vRVWL zXG6L5@4@S|Z6xO)bY<~aq>KF|?C+mpu3BAt zFdN_Wp`CFIWEn=X+@AL=-I{>sQl0vaHT?n5jGgA(=85VO#VGb1jWuv^nbGNun9(!3 zzP!Ff3SO2wUePgPeb;#OriJ_I=fj$M)uh}mmcL0j3>)1UK7rNI80PVq+XiKRKuq$$ z5_FX%3Bg=Uh=q16y2!`?_gG&81qf>l7*xN7CX;U-kk;CC+ATi)muOgMT>L>%VyzEd z7Z`3Z)24HWD$J^y{^dM7X5&&Oeu)HWzV__>4i=!mn`)pG&}teonUig-%vzMwNMUoX z_hn%Ag!#p^MyQUcJI0(bfp!)iRB<%H2o;Y~!tIE~@?=_(Q zaqR*MT;35}n7xW+Qq#HvnK`?A7O=Z!AA&wiHx=pi+v7aFj^t*Hb60InF+|rL-N!^` zktbzfc`foitNdYoN6XH^0ml+$tfU9N8)AV30{VbN4CbDsnBYWo!uM!O?6>ZLXA-Zc z|5JCA*EC#GS^;|9qVBIwvt*JGUGG1kiy>UIj42jLD!^TNlW5y}l(`MF=bFDcaJ!b! zj8!0H%R2r5b>m!TJdsq;9%G3%Rp-TIM%*A#cGeG<3i<;R4vdzC+_ww*U^7SLvGE0P z_WhQ4?^RV1lj^?;kEbSGLT%62rn;=HjB0c$kxD!jMfTcjaW1f_7b7o7&jz4aE2kBT0WRH=yZ>Qp{Aw%6o;&??e7h zSatMng=Gp1!e}bek0U3z?snVT53ayyFnN!s`YkAwr{(U_+XF))aarrD8V>Ix<1rVG zr(^Hft4%ri4HJmWuHvilScQs$_@#_X)mHvcP+Uw;p?Inp-Gz}=<-d;g3=b{cYOz+d zc>)*b@N{&0B;8SKHbqO`$qI&b!9bT&7pb;%>d&2vqGTz-C4oO~YNm1Uxcz$(X)Fy; zEl16`v}%l~Mcz*~!Oja;CNU|JG{zQ4(B)^L6MH=U^+x!;5dOU1$Mh_h!U&byue4?w zHCiF?(zp2&;+Z0?f?ewPh~+3shv20zE^pDh-^1XG#MLQIt(?jlzMXVf1Zv^^ynwkM zR^yQQ{5m3`rG}IzGC#*A-;ny{0LRgut+nlexEg9&cm}qBZPnPr-4M^0S93ifP@>`O z=bB-2O>LHUkHFc$c0uj>!bFXlO1kNMWPR6*f@I|t0pmB~g`(&SLTa7f{%kd2ZmO}Y z3MB4+dGSccct3P`S5OfHNr+e--I6poB!4sdSSDqQr|$En0@<^X-injc=MCI;%%-8` zg`HYYlHeA8Qd=HnyT|>f(f-#%_`y*SCbk*1hSf#QZ|GU&#Kd+bMcW@_Qzqr&iY$)a zO~)6jSNCrD=hrz;(z-3Jnk=2aMdqKz0tqX6Ed1bs;BrU;=wH^d|v|!)P`@ynd)=dRKD0Er19@qS1t6ASl(tRekGJQ)$yKhUKpG zu(ks5JdL{I<^{yHJu4C}AS5pLL-aLZ!4-l`(=@yZmKJ>+V^Qnq4-+N%-?aUAopZgm zlbdn!{w;jPu}^V{CafHb1Er8A$@KKY8Pdll%pDRiVyQESLmSTjGLx0IJ>G;?v=a@1prxF4X!+9T@zJgq^W{qlluW>EvW<(yBQ2OA+U)7?t{i z_ozvoPBPi2bz{#Vg2wGIlzzYfxRoaSA!5eZ*YD|`e%QK580>FCupL-zMnwuW7aHDm zW3pRdn8>FN(j`D*+}*kgPUGaJ6vQ|_$1p~JdAv8g4PQscd}k>?A6+h|9M||B3=4h{ zmA~VBJ>1PAUOdFa{fv(JTBD3@Db9f9PdJ48brWz>4;fQ)ncyAx1oLU%w|1f~5TvN9 zyw;KRnoKf&P{plJ+SP6yBPt87Q>U@HQfUw?*ZxuCFS1iTXHoN!?s*xAYToRUG5EZ; zYTdOlml6xN9dv)4&ZGA8NbIy>FbtPtv!ucPl229Nu*nR>N>BY)<|g)S!nxJHNvhF8 zFa33sCcuIWIU5uEB(b)|OqK!!F!5cEJD|y7ao9@GSt{eYceoA*to5ge>mF7j^x+x2qWeAeeH*R>8}zu zQnM`%?sT?gB-8-O!fuS&fzb@sif<>aQiM&}HR^D@0WSsloDs})G^;eZH-xA%sB?=@ zaΝgdto++&so7tr$R`P;de#u`qOL=6>Gk{SlEF2guNI49xqs5eF~t zG2%SCWv2EdQy6vS11gHV6W*&$U0DYXkzW1`=HW#OYSGu62a z*v_BkS6i$*u@ohWxE7-NR++4aZ?wjlOW)b9*b4fdbNoq8^#3H{K7syN$_Jdh0K59c zyj~?i(Ou=9uzZ&V7CWL!UXb@&S%y3<_?GF30UU>J@3@C<($waktVOvG&MuNd87uRN z0TBPz6EdmLySLJ*&^gQWtk9La$J_V+mL6YeU`yRovLNSwvMK!0v$wHq)0t2yCW4~P z`lVBuPz^&LS2uruPac$3`{tFM3OxV~$uAW7G{J?1DYiQA;q_by&kBS-M;A?F-p}B- z>3rC8=@z`}ZI^5WR;`f&8$%({7NA-9FVvB0HN&idfCjwo%uJxBj28ximG$up8EYJoaD^l?->PF#kY`Z=K4WxCO*u1(uPIkV-DjR)FreauM z|L33L64DZyN$Sd~r}37}FzQqPkxy_rxHu|5`>hIrFsC5l&4lk8(lz)mpHMM{KLTM5 zsPH|d@!ai(T&_v1a9-SHVH@uL&u~Yzigu+(#LAnVdeh?n7w?E9wn9S>GJtYKRA}31 zqn8`G)TA+W7SHf>w43CBq^v@n6PfMwqCiM*U0celyr%wt0Hi=$zeuk-fZq=0I~QyZ z?ZCBr-)QG?61FeEMapdlU%d80IKODgYjx3FAY~)pQONEXWQTA!B+6Muu`S1Sklp!M z$mZLR!Z1ghuW8iLguTUZRyX|4lxqB-km=6L?x#{NYRT6kFkvGdGkn$2g4qscSSKA` z-zIDx!pQ~3&ZoH52F*S74FDm_OsJ}4jSSM)%AvBFX$M7sY)}|2&b1S8N~Q`a6kqU+ z>)HfTDWm5~BU#<>v^Pze-jdZ%yeL@OKW>aZH$<;IB(e?7v-#?$uK9!tvT#8y*qYvO zyam5VjWCl?&C)Mmkz6Qu$#QuEIii>b$nLrZpVNQH(#3SUQ6`nrGOzPNP&Ee*Jh!>~ zB}-(UGm&;G?AG6d+ofUNt#frnxKNtWv?E@3QaL7-V^TT%XHq$OpGoB)lgcr+f0N2F zsT_M&InIP;CY}d z`FPXhDj(l^|DAgOz4rb)_x@Y?e^E@e9?Eir-k+CJ%S3h^yWPaq!o6&Eu3#&v zIn~T#jX)A)Qq8igYm=bI6$OZfoT`dKmKsq|JPDnf`y<87$)d*i=pcIpU~jopaHujy zm9`(0EIoEgStS(HCw>E!GFM*bc6J5KZT>RCWYy>@yAZa2o%=m%C^^$5Pt=@T)Kgp^clLsOL9X5xOuWUnF4>)aI)6HU)}`){j+dX! zpU$6NnID=y!63H&<_Ldz)#V}WgYcb^EvlXGboD>xMd*8=)wJU9YJLUz(gAmg~6T_Bv zS^p7lXhn18`P@2z2)8U5ctxF9JU+FisNe&t11R`5dS>vPp379t$!nUdosZu15&+Wd z=NFo4wyd%Xrag_HP=IEXS=H&)H6uyM70%D>FZO@UU;pz8Z*ez%b5*yx{DkSmM_jja z{G>&sqe(lFK2+BbU{Ehdyyt;}_$#hAFQw4zE|{_3ew(aimMT)nbe6FjQ1kJ3LbWdW z0ss)Y0SgnE=Tdk}#ikNc`;_9s-Y?N+k zfk+%tu=JXrEjk)A4fjd?9|d`aJ6zsEfY;Y&uWI9)`Sz;|m^Shp-%?DkcuE#vV(g)5 zl>8&XZXr6X2GVmo-geWqQFqifblgMq_t<{H%A6y_@Pga(`2NFTiU1bn?X0|=&1SQf z=+dX~cry;Y5mJ*}rYaRC7qcX`My5ocgvk%7SjHT;U1V^`os_ zatsnZG`OZWeM>7OVap{;0PutrYnHQ;X6;U?Dp?b%UfBEl#G2r9TJ$QEB1OYY%1dWu zYXn$w@Lqh7QorR2l;i_=|KK#cr5kn7XLvbV!U6FBj@%awZr;G2@1fdv68t?s1UuAg zuP7^V8qM(yXEBUVOY@yAjHAm9md0`Zg!;}ZN|~^JoQ!XcdQ=(#GH)(0Zn@E%yrmmL zGbP>Fx!As6=VrDCm@5}#A@zFaS~knLdr^9Zy*`?5O*{Gm6$O-03lo+aJ<?5n_y#XTtSM}1NRRCSxTl(^lGtS>T{r?y46O+mYFTw;W}?V{uTQUuf+BJa#L*e-F*Bj_Py%8(suf8ZohWl;X3-Y`X2Da zDwZTNFV0K3?9B??Ewh6k*|a2gj?C^A!PPb#3O#XeE+$vsF|lG>rZA6t$Spv>U}A;`Z7;|y+2G_$3Z1`UCI)|UYW5;g?x8f}3W^wJz{YRQcL zdS7K~l`HzPG#%5NW)}rZB1Pw~-(H-yKLoyHi&`=GozT?!BZp*XwpdGfO^zH`$cpQA zwU{R|_i#gIim%iYyGLiH*M01lfK-*$HS26)*t$F&bS$ZR4%2Bva2A6|7WUirpn+Y% zQawm#*lfNc zQ<}=mI8de}=Xa)OK0VK71LZ-an0Y@Bt!w?^@tmAB>cxqfRlfzZWy%T$Cg6v6OyA1# zIy4);E!?OqWfj~t!|*e;i)zF%09Td$#PDHvH+8zM21eAc%DmbN4{N)yhK)wU&aEzw zLQK$E^W5)d3uFe}=XQG}E9k(FHe>fRNn|B-L@C{ioe#R0W&5p;9$Sm5x4*oAQ-~Gq z8AVTcteN1p7JH5JNP{D-jAa+g%QTM%%oikzP^mlbiV4V_Jcb_SNwfC74 zSjm*YI<|jP0_&8(y1xY0PS}`GhUP`pZ5P)`%{F(=rVrzxdVLTR0Q&wON~# zE1N9u@a{VXC54uAD{L8N!4s;kNM2={qj!I?adJ8kzH-;6mYQa^gnk#@B?jHtp60U1zm{U>FEZYzeTIb|U zn2xCK6V=lJY>sh=S+%g2@Q&GITvRRbaBw<3AH1ooGKMHbT#@0O% zTs4_uWIe>1yT4Jsg&tbhaN}CveH%AcOR52V^p!h^yq#cFNh@OAbtb&EL|{Jr%Z&2;lI4X_U#@!d z5yw@(v3uePHld#UhfAFAp4YI(le3g}Lo;>FExzq547Z@}Yzl?~GVWM>Z9qVCf!;K^ z#^|M#m!K?d@OG{9&=om2uwnK&qhd&eb?#Lg?|yh#Q}E`-Z20Mg(>Znb6E_ zn1Kawg1U_!H+qge-!*m#WH0fb7|C3NG*d$=D=6d%6lOD2b6D1TlU#zRddKrBx7&OY7`Dd;%X=H((g1OYj&Hn<v9m_`9EH4cgez$2t7R7sr1+{@MNGcgM#^ zM?ZY?_Up^@*FW;(pMMZlevSY7`RE7s`o~Bj$H#x&#W8bFOEl*~&ty5ngYyEyoW^Sc zqybuqy8YbC55N9E`3Eg92OOnZ3R$FmZ zUnneqIT(Q(!swCUvp>TfTf;9hK^K{;;`cQZtmMho=z6Fx88wm!Sy9dIy$$PGpVfEm z3Og~|`RTrwZ-<}|n~CkL63owYB25|n}%M-%qZ0`Qm=S*=ARISUHhEcDu2p(*rIz0or z?+0eZQN8+$88QIi&HiE$jSy4MMk!xy{S_~^0iswW&gpDd=Q6!8sxMBTy0SI~h%tnu zk}x6(aRKTjGrYo$8FDI7yE4+c001)=?rn5XAUr@5GAd!7jsU;lj^;(iPKcJ0WKygU zUG-F;gUXyqoKxM0K1Ez;I`jguUa%|MP;WS%FND-zxa}P1sQb_;r8knNq*7pv6r>X9 zD@bbyt_{h#GE)1H-SNUPqk1|OMzk(6nOw8fqHBVSt#0A`yO+az->OEKLjagqMsb)q zd^t0LT)_paJ@!1bN9#E|skil-Vxo4D0#P^!RM;an1g2XrlB`E<7-0Pm^hD0dJEQkz z*#=pF72AhnlNXE#mN2Dgxv49Bv z3)S!hv$8OLFsDIQqD|DsvZ?{Hkiuz$+uIJ~e_)7%bv?vYPmZo9ssa5~ zfrba~XnEn9-rQftEb6IrlH`A2hi1^+>{+<1euUEupHbEK!wxj}7!HJ@;MlckzHD;W ze2EqhSWKyYw5q=#U$jINE2aussI?J1fCrExW&bd@i!i_b6Rc4C^`UGr_k6J<5gm1& zjpGRa5uLLD2t*?EUa%WV&nAsEwtfhBdNAe&Np17 zwG*<2p(vSY2YVlqEZp-|8!~6j%XC&yNK)SvOr7{Wk0j{~vvFM*b^8#2)VaqB!-P45 zv-L^Tfr7AN(SvjA`WT}N0UR{XF4VzR;@M|6x=>(E^UPqCBVHRoIN0uFP|(L`OaR69 z)8c>kmw62HbSE8`)Cwp@P>)y^YnYRykVTa_7~UGLUaz5}CzuCa2bINN?oizTWMmr! zW;J-=B27<8%++2u-!Lw6B%rWD>BbEXwjSjoq6s8YOAo$Dz;od`!G(wL(CS&P5!h9g zB#hOjHpgAs3_EiBHL>zaxYaTiA3#_l23Qza6NALCJ5 z!j5xCUo&#-0vvUu%G=URU-d?C`OktT*S8RUlgPZF+6Ks< zpe*GJKH~6DN2OxBhl_2=&l@f|^5$koB8<@`oFwWp;#o&-Ha1}J*{eZ=`qeBeoDKS+WiU!IJ2frPJPU*{*7sR zOukh(vGu)Bds=(6?FiDoK5o4jR^gtLqtmQdd+mCfXAEWNqFipxN2|{-OQ}?iWU$}v zPkQU5w@!L%`}tSaTRq9_M8{=v2Sbx8+NkNn5#gW!I((R%jO+d=5jJw_Nns5D2tuMd^=ZJXbdgcUl{uulB# zGba}eRgCs=S@YG}kxOVn6Rt5dq)cNK zbWEC{x>Ak!2(kw_;rHmItyg^CQX)q2#$7;TvH*9!9S#j|Ke2)0-CsrDzvT0b=xsmu zitpWi+Wh4WMsDLD7gn`#0q7VoAA1ik!R}S*Mk`S6bgXo~ ztRrOEU6JvG>ulo-?4bSLa=?y^-qeKWu1tlQ+?gw9U=8Z^*ar&)Y(MM{oFF*7-xf<5 z@v`bgW7>XNySE1XKmZp&1i`diNE5WeLyV9p&DK&pWhLx1>#qvS?c;=vIINn%AxB`> zMDDC^gquz*n@EZSsrTDYbPydGiHH+Pl58XA3?byi(+nW#L?GsDKjStSJ^@6X?I#{# zX9Q8QgT@m;+}VEaaev1l@@zk`2tA_)zlG-0gV-~QWB}b~6oSw8bFaw`Bm8VX(TG2z zCZ>%J)QbQ#ie>~_P=DQ|ev%RBLHp_E4w}#?vct$jvFJiORLAgqJHGqu%Q3;!?uj?2 z;WLC*wEJ!V5@-7vskn_DZxHQh`+112H+lufq9aA?b}bC009sNc(Ez4WCwfvO={Ss~ z8ciu$Nr}c>iljIe`f&SsEREt}A&1ry`7)#E78q;6-u}L$dO^hOTzFI(M~|y^wr>+_ zVw#CbfEjV5k@l&!KvwGB6WUA2%X5qor?5g8OyGB2B)yD%uIwB~kZt4EK%~UQ8A(fi z!^&2|hb(RC=s3?soo`L+?(SAgED@wOGSISf_*PU}wZQpJmSu*eVrk!{eg6iRrh8hZxu4d4Y^N z;a3ie+pVo(Lc%Ztt|+GGTNT+NRn^|4NdyyF$V%8foCXWkx@UMM&A44Ib=~mI5S~P9 zTAMU1zBHqEnbKC?_k3ncR+h1BV6o}uP9&8K*N!Ss{%^yw;D(9l+;$vWcl=!E zo*Ut{y;lg@=)3Q&Ovhe!&4)EOT?_Lo`sDClsi7{O`4!KY+9(Y%%P}N#WQrKxy;T_( z86cod?tdchAvH7~I{C}mb(_GHIQxDV%Y@MPdVfYr3elsEl61pL>kL+-_vc91+};KUg7w=b1zoVxkaQ6@i4-@i)DT><68xvZI~sdXLxF$+U4#1< zva&(c!z^GU`deI{N<`6iroxUpZpF0=$xCFOSAy%!6OcX3^&(64BxN@&d!qPiM$2T) zHA~Dr@`M(A1_Kme@SW#rOteY_Cm0TGZLh0~XQge4xLza|5dPuy#pU3lT44Z~nvF7S zOySMKnyKevi6O#6+oVQgVj_jQj`kYjMz2uS0wOV7=mF7kPF{xGqlQl){0`L0d`WYb zy`-ue6k?yt3K7sVvn&r=*f5SvYjIy&Os}`DG-YkDsf$&XEMqr_Ycw!%RMfOS*oE%D zvf><19+1uO<$NC-V)Sjf#p6719rP|#7VM}plF&8V3|8rOPfiQaWNVZs|8&qAV96>s z4mBb)Bq=jsZ9o~gSjlwB*$pGKTyR}d0LH|1gg(|K(16c86{?N2Rp0R^gl2jztJNBl zi(`Y6b81u82zB=%Sc%{&#Iz&Kplie{S{SZe^056749zWRc3x+N6eZ7T$+HazRZz5P zDN8%DU=%)2SzbsD$`fqaRv9b5Hq^_2u z%eDdC8zI{vU?po>(nPb8E6o!`5DqK#c4MG^1SrC0eb=(wr^RO5JO^s+z_dmR39)cu zj{zn2Il_oVdBR893PRrBSlF<~fDHQ;;KIg2h51+xA;~BPCaj&H8K4-123(y>f z2a5v#3bfENq(%aN1v|Y9XcYkc3Q*hu9NQD%*I2MGfbxpi273eg8iDp0ohbb{Nz(gI zkbaEh=rM`WBT}A6*umpcokylQk4foFj)#=~K?z^($+qoJ^Nfv!R_M*s$^3*;&^6+;kM zy#QCilQ0N>)dOx7Gx!kn)mF$sNXZDpUhRNdg~W*fW(Adq4Nn<$VZ5B?~EW*8u;ANnYQd^b2!KkQKi$k0zfhHO~ zvEv|#qTz@}Vs?=b6p@F)5x*__VTiWdil*5ELDUT^6iu}Yey9^tD4ODfpod1n4&4KC zXdL{`7^tCGn4wV+L-&AOi5c4{SfTrBKBHD=ER>K135w`OB#h85Kv2vOgAhX7;6QPv z;j0g9@Nez%41MJfa+Y@M`8`LKN*%LgJZ9tz0xSjyTEij)r zsGbP7i?sno!}J75Zf6NaL-YhE6ajq3!1Dx0&wJvQdZBqj#~*-u#=-IgmtFJh^M&Mj zJUE_v2i=UiiNX*(_kiCCt68Ibl2c4yS2jl8eppmrj7%Y$Kd_Lk)d2RJ^@ zvnW|&?ywhIQQv|Y(ZKNk&n=CTCc2`TC5w5I@h<#kSrEi$P4~*DKKE=&BP76bJ0K0@ z_o0=!kJ{T>&F?Wz$-8h%mN0(9)Oct;1>qAe#`cJ;PboUe3LQ(jxu$De9)&q@ z=0_X!7p&10Yp3{6HU%HT zJNXS0=19%SH_XQKhOSWw@n1?)krI?jWRqMjlD_R{Z`D1k}dfi z;-{z=WabnBGQ-==#;}CP-!PFm`GUNxl$Kr)5n4Da#qQ@)ta}ObR?Ac$57-?w^yZ1o zpX8fa0?pPQzmEq2x~By#U4qd<-qePY86tqx0Ya6idl0n6LC#5wKicjCCFhD`DX9Tm z!T_IZxM_`wY%@oViaRGCMd#U$0xGeA<#K8FS&d!^yWnMg1V6w=J`?AqT$M}(`5M5t zf%FYh4W0v^vnfqW0EbfiCwtYzDblQ$o&#x%Y%w-L|h%h4feKgwh623Q0 z?RMPo($MiDnw2>hG{Z~-)5Ui(W!O2ZiM-gFqp&`@)e-0@7hJ7z7uEAv-{QfjwPjTT ztF6EGC14^LOY4MmB4ldjIzX`*rngtNcY`?=hR@?3h#7c$WbOHEE%vDCG*sKm?RsW0 zp|+MWK=ZPsLIKnZyl%8vVH20OdyDC3mTiU35?>N_S5TC=DF|fdKFN(dR!0}ke91JS zq46)6m;Kdw_KTZjjt=XU(T8A1<#47Bz?$j5_zUh?ta7ibRW{ZX3QIoglh$|KL* z_FnyX(3S5B=_E{_5FQMTZQ1uMA44YvmjcvdVU#2ok&~1HG(9=$qadd~TePK_tlVr4 zOB5zMoOl=Zn{7xXu7$i6wd?}guPnQ6+Rz2#%foQothnohLTjG|Y!Hfx&?^!4rg$8L z%-U)L%mH|9z%#1~BW^6RV6?ULu7aAgw4}O|t>K)1_cCPvg26iQG_WCC0sjUBRxc7% zDVcB@j)sG^a*GcpZ|m64iZ%7Jh-(uT*qy@Gzrjd#XV01sOOp?ET&+qfs*DbampiBY z5%kiiAk=XIsH1{UKik052L+u5luku-Nt0{UGOGu5IY-6Xxxag9?T6+Na|pV`gFLC6 zlghbIm9xofDP(E(A7s|&!O(&QBN;7MEh`v=@AtqDIt1QQK{< ze^ye{6B|xghM5++C~=*3kE^+4%sm9=`Pc)?m7T=1Wfx6fwi->FW=5*FEx>&a111U0 zk_w{-vE8wA>t!(^ge6y)FU8W6o>3V2BS6(su#hJfmqEy;SIwHXq{QImRimibo#kAa zZWI`{Lo*}MK__CI+d%9M zN=$p%U{c`T8RNNZK5{81O{^t$nT#aV~zI4^&!4H3_}2@aYlqic$;Er%u&dyLw*zZ!7In^(SxnX9e!H920MJ$_< zp^eyd>E4smCefOq*ec%jycmVk5;O9m5p#_IiRMHMR)!1ovzWh}DMeS~C(VB6W@=sK zRLn|7Q>dq4e~p?qE>Zy57ptw4WY)!-%b4jJHT05EHB<+vXXN-9M#LjZK32SI3P?l^ zo0|7aeL`}YthuoF2$~1X6xxVL5W7|pLa&_w5T~JofKc!DR38M)2-yrUe*!z-)LX_@F7G55^kA|3O3>KP_IkEA_f z3Su*KW^pkeD6mFFr9`JagyNH|q|_OdIWn{_L~2Vo3yjlGfanfKb`fP@q6p?66ZAY) z0<0luA^`iyHgzO!gTt-{*gXSIw~N>Z!Tr1&lRRSS-|hx9Zz&mZ zK~D(JE0Y5hk+THSeZs*|1Rs^FoIi@hBz!0ddd!%B+vQ2ewm`U_rUR zknUu5uTun<^;Yt%P74``Ac`P4t83=E|Ftrf^BDGKNkkB@3qs;Sf$|6ygeq4eXS$We zb^^Bj2%=PNAtQDl^E!zkg2~}TG;V$WKRc#lOo>=L9B%k?XlBVMn&`a$tec6sY+Xm3 zEqF~$Ll8ps56qdMrsH_iFJe*++qqVAatVO~GPAhN5KnVb3lH}#4lEc;l}$zPYCF7> zn!O zx9@ppdBY$3gz{e^Gh^_Dfgf?l0ie+CWV9yWvU#)40HTOnS4TQl`GS=I5RsI$=-eRK zpk>N+{vqIO8kX{2WE;z@xbTV{yIQ5wDZ|BZ`W+owgGixs(5?GUatNNNHPi1S;fO-%e-;R? z3wX#w5qka4ie_`ef@)AZ{8KMo`*90}!+dtjvoxV4hCtY&;!4`E8pC~hDH_Sov7Tlp z*gB2{Ej3T7jFzsL>*n6;&}`ESBZ;KF>Qb9|Zhy6Xmlv$$5)osEOz>L=hn8X4#IYTvRWY*iEOL9B*u4E!bAb53=@~a4&hqO(GxH>3|=(L(9~F*$_?ec zp3NA0=9n6xk5e7=A?kug1aCY}Ezr?bKy8K3LAc}Uf5w}xqiTLWPPNZ5bv{Sf$>M8! zK2}xF5%oMxJ<156dsp%tv30T4J0GKNXMDxZ=;C-$wL0q>oDmQss&uxhZ^qn%-uj&T z*4>P*xHA!75R8?Ovh){LyF8EJ)$nSEph&GB)#k^B4@^)er)Qs(#?nM3uOMpws- zi>Q2vx|k6d7V*gVDwy^0{Dp1f6ojKUjd!f@QW_Dm=Gb7lL1rB^K!Dr{HSAHfM@30f zC)e2BRQ6%-c_(=e-}OR}7-9B$<*4kgaI^D_BPXrrK|S5q$^jcF-84=H!BiDudjgaLZfJ zFwouvSKSzDSDG-C&4X{IA_o>;b_Sg2sW-RqQe4UOrlBJDpL=hmJ#u3Iv2gab)84d_ z0oESDXoeYKgHq`Yjv-Yau5a0W6pcE5gAccwxTI`VFxVTsUU1z)sfr6mD#7lUKK7I= z<8)4q2j#uS+;4@VC09$XH~8OP%D1w#{*d?mLN`_n_1h=qT30W%>Rxvsb}fc$e`LS; zY}qs=*sFjp%36W9d~TJ=191)(v4&>aQF&Q}m~f))jR4S!^X zv>AIf(AI+ixQb*9f&r)?Z@74g&o_#!oWa-07qcIwU?j;ZRkEW0m@FD$bytlzPf=2U z9d^Zhfvz?PLBClHt$OJmVrLyEqrd%hPX5sgQLp4F`3L;>nfdP* zhkeoyY+zook&;E~n z7coQJLMc>NS``-yS4^h;Q#S6g4~k>}I+;@RFljg7m(kz%QP-t(Z- z4k~2{O7HqoOE|t?vvw>pUY@G9{MVlL2-QeknzMIBL?1 zACACNv@2lW3H)c9@7(ZLEyn5*DcKR z`!;=a=B$R?1X6S;6+OtzJDB#fGjA6{Gd0iuI*P~FygbbCo*Skd$)xeWL3yl3(@yOC z^+~AT#phRAafoYXpbrizX&0J*0-0YpoRM6H(CU0Km6|ho)lZDRBKv za!0XqXVRh6HI%o|_1ssfdl;T9WXdj%Rhzx<83?{XZOT0iSZL&Pc76| zXqobhy0;j!g1CIJPCVZiy0dL$nVIkKM}O$&iQvm{jMis2z+*TXP@6$l`meBU0fAa$ zz`S*zW*NI{2gNaVZeRLf34W(N*tV8MJWdoxOn&j{GU9Wh5!cUhI@YiCeZXk{3p{)wgdV9*eWhL#=NhX`x*jaG+ATl zIDX*w{TD|c>(Qc0Zdqx)NX1rvD6dMgpcxg3z9{V|n?UCw_&7p@Sz&$xq1I9cpY&o- zxlpD4N=UJz-a;=jCC@auv4D4tao9Qv{@t(DfU;tkB4c$fnDcy@b8j*JP_?->VVx5; zUx7=#g<40=o)c@SN$R}3(B_+!;lbLDNa!@}@S?k{%AU7>@{pZmE& z3zc18@jECw9%JA`W9XV?IVo6L1=@4KB!Sys_|^c(H*iHCj~WMggsngNFL{G$^Ci$X zEc3P`?2Z*%l1qiTqd?-RQhX&gA>aPe#-)J<39GZp){)V+j=+i|8|3*R?E#({5E<5Zbfyf54rp8T)G~5S$3eUif{>5o+*C$_` zKRbVVqSyTFRIlE%%s-nwJtcwqkqwAuWlS|}^OwSu>AROjYGTOEA@+hzpj)B!>~1W- zh3n_t*n0`&23!ifB){6og8ix==^dP2+j*bqdw>Lo;+AIC9N77{GRTf*yQID2seNPN z3z~4TdLhM8+&G z*q9gauygRR54kK%qFO-{rsm|j%1s5`U>cs5bl=9T$ovRS|RdFP9|c3esl3O3zXC7i^6w(;88nwXytbW|I;Nxu6T4@p4P3Y2E=yUzgDYIj&d1 zd1Jgqlq@qzP2uz>q+A!WTCK^`?)N^i`mlRcg)RAp$ts4hyfM*4whp9WG`4Z|J>`%g zX>1I%?5%Z!?+tF~lYiKKINjEJa(eImJN5p(_Wr%`{;mAq)#)EjySHvA*Dq6jLdwoY z-MqdgMl2~MTtcv>DpP*QFy-#$57M`UM)nkaq2Vk+l_s<**Rs%e14tW8=Z+P83B_xn zRDUQ+J*hzj!nNbFc2H1zL-P>{PA}{`HfS2kW73k^{f%qsV=8L=bh$?+zPJwIebDq5 zF}{q>@msI_jp?*oj_$)M?^toi+5P#wEVRTs)9i4D+)bTBTZ{8w#&iYI6v@oEU+U73H2HWm1b}WXUtOD_e%8 zDwH}ORH(F$jq6*Ue+Tx*$aTF$ABW92^+ipfMFWy#4CdUj?O_{@wIV)=eZFPe>j7>y zz{$GSH7pbl&OMIL4f420;%|HKwn%C_%+o$)em0huMR2OfeT(Bze80MV38ABaCX zGDr^P>udWA*7{Fbp2@8(BI8N`RjTuHLBP4b$i)WpW=ly{Ow{#eJ^0rpE%kM~W+nMo z7v}`8bwJjH8uD()zHAF) zLlZ@t(ObXNxdq;2?y0ijRB1;_K>%B@Q^^}y4cAXS`p$b66U~)PZ>{$f$}qclla7h| zP++4QU|pg9JZHkxCPkv@Kh^=(2o1-%+uq|^I5?}@wOcevF_0e9kGr?{8at9}f@ttw zb!={2NqukjUf~l;Hu@KR0ib%d>71nbOiy$_trxXV#e4Ximc62)%RxxP$=HK7!WJ#P z>><-{Z=mrt^!R}Ss79E8>t#Ta+ zHU^#w)pfGvt28OsIO}bd2_ZA_vvf;nAuC}D)$~GiHMQTgCi>RvA8+!PTtV*zEBj;o zxgbyHPoM0)%^?}AB~WsO%Bih~%ZL@oEROFzi-OVg8hl0ob@@eDhHm<0@ zw4)<(f=?bRQb<{z?tOe_nXf+NaRr*8ExYCedCZ!7*Xiv=V9kFt#TgXe@QfDOHWY_{ zjVpm@b|&t4AtASd+|dH_0*u~~Fh%~?$#)m;{`&Uf&8t&QQF>uAyV|-Bia&GjGwlQF zN8Q%J*?H>|0NP~Np()*_Dl3|0Tf#m-8spu*_jDkb-g2SNziK+uCGhMQA&<)+l5LPE z#Y#06XvZu~<#bt&?bW1-bbCn!M!d)^+&rSJw3MTfLJg(W+*~AcX~l2HUyW)dpf8a;d@zvvzOBS~e8o z#ks+xU($QUaJdV;T>WR(XMcz>K`m8(>P6Rf+HoS9D#A{0Wkv2O z2Ji!=nUfoyKPRt*s<66G+4{_&eFE~+To%Kk*4k~uO|i1QM93>lcpGp;9kCDPmq%yh z=)+R!|A=y_j^^ZYlV?2PrIu90k`)DF%YvB~f-w2g7f&7qDT3ceGnW>gQ%UZ%+|2GY zM}*#xuIQn6x|(?hAwsT++;Fce7+jK&Z7j+%1}WMhU4iW2sC&htsrN%eBnc+Z>(POr zmrEkC#N?T&hKZF2xN;8#K`yrT>zV?4NC*WM&7aOwc6Y8)`t%H@fS5Aiz-g?LDsuGn zXil!Je01io+dUVAxRiyP2qbAgIU*;rfEguJkc?5&Qq&NdEtY^i4Yt#aDWOB6%IJ^? z9fRy3hke5~>?tafII)yP!i*Uj9|`V`Vr6M^@0xa!k4WH2v{4qJU%rgrC;JBWkLG`` z7`c3D3B1?BKjrE+xKWnr>YdV@_~c?kf0Tl}`s1~k*6B~QX+g>}WX7JkJq^m;0XsJR zu_~CBOQHlF_jFsh(Z7?Iw4^BG00j!=5Icsb#g_#%&w*DwA>{yhA_20$dLz>jLWTE| zCE0)0Gkk%+lZ}Q(-b%s3!2|CYO`%=r8UFM8f|o2vjiOkPM)9G}y^RIg7wlqu9~T79 z=m!ih3NwpLE{H+)hgik8?=D9z;@fwZb`AQUa=9~om|~bvKFst-3aYi7`gKc2ty#HA zgbU7|?#Q^2(q$0ps}J*!Ay*!{=eJx01528>SU(Nv1>4NC9znrcEyHdR*gJWzpz zSd`ZY2K7@d_GrZus1$k0uxLv9-1f^D#6UpTPGE1a3!9;?t|^;F@?nT4Xr>hIHRB}0 z)J(<7)47!nv55n0!@<;Ow8-Nmo*f$~InjSR2ixlD9BAgC+_PrIj=*GJ>=lX*lwXE< zv8A}d2flrGc@VD-1!~$dRBNv7i0c?H7p;B3qBmyYSxb@jL?DCKx!p|gdu_C85e1Yo? z?tjG2ao zy}Ee$=GAHOnM5w+o@vD5*XVo+TJh!4#M$j3p-G2A8` zaNFMN`km%(UpIaq(jIdYxqHjD1o{*%WubAiXyOtp6WE0FO0i;&pZ-)fcZku!WZ$u# zIxPKL`;INUcGmpt{J~*@gA!OX$Iprbom-9VCV%?9jm^P{$xr!|qj?`Co_8dt8BD3z zFQG!;)p(@WPiq8}K8q@>tXI%fLX=VgCm!Zf9y-%>!^I~<4Tfb*E3)h@KXQ@w?|RFm z3C}hKNb{Ry4QzkUYDvIut_DQZDQe?y*_#J;AEpWL)w(o|KQ#7c zJSnjpERs$+&>V-@s6U*869ZwvXC&jdj4aik3R&fKA9)Or4Z<`{MR5`}E-F@j#5klu z`()!T!P6~?&~&-DZItN^H4`T$mIFdRFt4P^SQD0dHBxN4R$j=z0Sbg2$--DStg@nv z&}|!*c&-B5PDTqMp_T;%dwl>@y+VkkYbEVvTxsltzLtB=xQ^7;n6D6*DzO6@`{#uX zXh^SNqSUs0_aea|nZdMXv_+sIPRt!jBPdU?vNFd6VVHW(OeySX8-%%`#jVy64MmkLUAaBq6dOs-%Uv^ZNIY3=m5VjJ~Q?iZUoo| zWyv@mfidG4!=5qkIA&j)M9y4zU!bzI4pIdpAdcb4!(q-i9+)rV*poZsh~?2Z;y5*q zArBvp#vQY6K3u*bsYn@@Zqp)mq?BX>V!tIKbMnRzWxQN;p=4H-2s{QE-ZW`?E%mVs zdWa#hc8>uSj{sZu?m?nFMDoq@VhPYZArdED4H}lNT=G&IS-&ijoyeny^#f!oc!1 z+pr?gtl9=c+MKsF6{%?vo3H{13Hwm8LZ8Kp>r0Xr{EiihocuAAx^6lO^?LrJ%oFgX z_&{ZN+>&PDUAcPNxJq2&kc#RoEOU)-ke~^qa4W0A_`GcOmqZp?=k7~{tp`;7SZNic z@EQnXTdsq{boUF5u3awu(a6CynX)Ukw+_lS;Nq^k2jx^5Ya+PHhq}w)@Qw{tERc>| z)rUsfRcvaYBs9;npz}RyuZ2w6Rkg^tT3?Sg`ggsI!B$p^&8)c+<%88G|N=G9)>0RVHsV61Qm8+ zF^y{^v^xT!8z@!Gu^llR??h2#BQg0@5GgExGeWb>ey6S%5T0$p%moOc-Q;Zf?42$9 zM^RhN5NUudeKJ(~svedRS%$TLINq2%!i%yj_@XMqC_AQ?nLmfYvutc-dqyDFXw@Ca z*vxEj;$5={3$?qryM(b83P2vf9|Lt;V8Sw~pyMLY<7SgSG%XCDOb~S=5+q=r`--)x zychShNH4B>^T(PZS^c~(l-i$({+%wMEk5Qj1RP$w-m#z33)W>|Gi9*bo~$5Bbb7f$ z<~yPWxY!;}HMEokp-xV<$*J~vIMpuEsirw;#uXIRY0w*6hAM2O8|$}tSyMU`Z`)~K zo+Lgb$w+f)>r-r%DbA(oocur@5P&tHz;vwnxK1I#X5Q!=^gNKc95CzQ0AYs;A(1nx zGwcwq&QOcj_zIF`P7@Za##Il;SR>5K@f$^;x|0=8W1enTV5+N%mZl5-hQdF;*8A2| zVXJF!MYJ$iGAw4oij5+2dDe)_JGy&1pu5{K!YxHC!i0+7HOFo#JZCD`$Wj-sj!1>C zN9}-xYmGbm2+NH#P!kZZmeHpeU1*;i(BOmY=|&uoKYSS@>U!7i!!u`zZMqK|qD9Ot z92P#>0!9qfC=sM7W<-FEaUXil!5HUJg!|BwETWvVG44Z8zK9|dQ4&Z)%7_5_S-THC zrg>#NiMy`Eh)wYA+py1A9KwkE^SWNIzG}vk7;{&PhWqgNcs@VIz>ivCsLJK+w}gpA zrmSub zIVt@SP<@PuJy^W8IfF-+uw&y_K3;UnARus=z2uQ8jP~hf;zG$bxh!b0C8;ruZAa9< z@CslYT=Wm-m-6C}EXEB>d9@i)NCTetcjO+5)SBYAHl_j}3p;U5QFEHyqL$>g6C0~| zdA3JK#t*(>bjTKnrf3L+kSWvqZ2{q&8s=Ygp+?-*69eQ_%ZYD5OnCI;Bhf_w_>b(h zmT`j4s39&H&l{sTVk_a#PSZxJloIqzQ%+_{GMUgUqFf$nN-=;uuk2m5jzEn<`im(2 zfmt+DVS_i#^Nc4@5Fa#EIA(`7H&D~}1K(5|QYnq(QkZ9gP|b~T6TG*z{Gm7s7XYxS zzYoa-*KQ|kQxIg7TJ=&@plp~Gp!^$zD3{>gcjBgU714Jtqu^QmZL|ff-62%M1fqSE z>+?yS0p5Mv+_WTyY0HTGB!--0bMV5R@oXjq_Mn{=3zWvqk@7Us(KWAw~M-f86gL6C11 zI$HbPGt1|Mo;I>W6{@d-u@v)TGUmV+l~8A7acV;Z`vJ=(5n4fT-5bz~R4~24P{*c_ z8=?fw)mlQYyM*R6;bowyTtSmt{VMqEV0vdHIW^(U{;@Sve9Cnpt5v`JM%~*1Fgcy~rrvkQ2oid1X-=|8gA`m(dzoG-Ji2 zYab=B7MMqrhkWj)0KeXW&Pkp+wCenGKROf;hh=*UFp)|xuH)kBQu*3E-WgM%I=J{u zij`k)tG=O{9SFjO*2B|^X2=L|^1UEmTwfwszFL>Fdj_6e$`!`g zyOa+=cvymGZw|uL`()w1|D9gAe)IRU-+A2*e*a!`oI*bk(IUjW;m38m=56nUC1aIT!42HhKb%%-L-dS~4*huxztog)B`|-6goE^q!o6 zwGfLRoRLyWpgTYdgD@`yVIR1HN+`Iua5NNy&7FXvKdCXn7QP81sx&C{DaEMZE~VtWQX&3I{JqA{X2JXd`J8!E7SSiur0R9RFB zoD?OCLf$grQWG>cxCse?#6|_+$=C>12;Gn*SyQoMbt?CUrhp@VaT>soHN&0;U1|xN zmvAf#Es!{m*AK$gG(8*cc5JthXT6MvjPO%zSV>b_(ovDwZ>Z~>Wg=c+!@NkCxSUO= z3CNs7T8Xz|aL%wtP)n(tleaR`(~1GvASy$bHLOwQzJ|n-r6jFD8jop9k0aVLtEIb9 zJ9<+m{wH;mE$l^?tU2tw)Bu?%dBJR51C8Kjqbq@k-Q4nF!?dQ^D^{vHI1~AqCX7aU zF_s%(c5n=XP^mFibSnmUh)Np1q`B|V;>Wn?upitTna26^TCLmam+RyUOzu%&-ogi> z+zuOnQnq{Ck~O{C-_l%5uQS(a2BtKF&&{63xI`i%3hfx_X^Y8MwL{aW>u9%7tMZR~ z*N}P>x8Wj#BHB~g!MG+t&&jVE0*(=eokoudIVV=dpPK`*wWFGk$2hD>>w_Sfb!(|6 zaVXB2XeECwi5Fl}w<(hvp$nO1tbpw@+J77xG^&_D#}#yaojLQ5Q&j}JkAi9CGpewf zq>W1evzLuZwBL+#9M9LTqs}UwTM`2A?RXE4?hO@(8-fG(-20&&{{V_u}?c zdXV0+rLolbKQ-Y;L9Kaj13og}2JpEHWQ_uBBzv!3q)BJ1<6(U}Ra^)|xic%j5>Dqz zA|g7*#&kd=EyRPxJ>8;47gv|?vegS$@INfKk9HE7}Z33y-njpP$&5LxF)1us>jpoeFJ{qu}HW~TS=$Me9Rzm)0 zoHuNp^_UqqtBXy5(~Dkbeaeru%9tEYI~uF>TW?s4%O|?Z_?Qi--oXRajr|7)Q|SH4 zMh15r7C=p!wDJ(SNt0UD41|hZ0ckW&mwpnQlXb!VpucuTc)x|#ACw3wW+qwRId6>I zZ6k=$ra(oUWM_mnn5_@~;_A{6=@TIZl0AGjPH;m@TBmY@V4U<}^T}g5tXG!Fibq_e znjZjL1Ve_Zl7vNGC1W=9J0aIBz--TkmdTnaqGS}h6T*cmX_nQ!x(xzIkT{Jm$rnW9 zNVu1MZ!DmBMvaMTS&0Ng9eKGO6QsAJD1cQ%p_Ps=(UVYH7ve~&MI^AyZIO$vZRc4~ zlub3kuVq|BVR$inuEZq6i2}=CI72?3Pv|@xHy09(Li; zczY0cN^YO96rz#G%!F?+gakb41&h3#8?Dq)XoI`7H6fhK)SK|mLB>f5zarm*58Z== z=G82d_llJF(pkMw`incRrJ+XZtfdYZxLUl_N0|)u5qA&)+?l@X_bkiwe?t^|!1Nth zJzojT?&r#yCQWk9Nm|G}j}??OuAjD74`Cq19a9htWRreE?HT;BVuBU41jnbT;K`SS z?o73D753f;9}UpSY>#v#rS^{9Kmc>1#sGVHBaaITL!FXuO$&I(UtC^xk9{Cc2tpeI z+6T~cV@w2da&nR7YwydEjK$F=m%7^A6@T7Q5MLBhskatvpBcn1A=bcPmpN(TlP2D^ zyh#)98b1e3+@oQ8FV#tBP2-}s=)Es6c`Hj3r~r~RCF5+MWJgP*Ks)d0b}m?H>QPyr zz&rchKNWPK2)RFuTIO9mOBHOztUYF3N@S^Zyav2~tW1P}{yStGk0N4jXXoUa*`7K6 zqMJcr(XchePFYELrfOb#)6Ku5fsWvXW46`fhd+DkmP>>iZ<(s|OTFM{KOo;d5bdz) za=TtG$vc+FJ60e9y!wz+p|})ZQ=Y|oge4kIz-5mGEY=Ei#rD2-D z(7a%GTvl=WVA2?>5+w{(8eM(OSFW?G4ONEFn<92W(SavnOgk0n8_{>vD6@c_xFWQX>=#*B9SabeU#n^{D|{qWmE zF2ROZmCQ?-H#A}Nm1tC5Jl{}3S8V3M&g$f#VE1KzjC58Imx@2#kDu@TcxTHzv~CD; z=J>;Pd4;<-q){Z-es7FSE;qa^nGJJxHv{5R(#ZOH{xCEF94Hvq%s#kYo1Ovyn)W>p zWMQ<27PST8n;csqz}=pi96akd7`+1tL~0WDAz>^vlmGn4&8Ww*1VCI(iUvO07foe8 zx?makd&P<^(Q={93W<&3Nu*FbHMuLsTNk4Vj1^PBP50`9mW`p>)n58_8%VP_A#6o7 z%DHXyB~}a;ZOE{(ZZ$nF-oA|2qeg@=k{wH-IrpwbL_;LFeGUdt3Kur&to7uUZR3rD z!~|ZOR>2l5W*juyn~eZDT++8W#v&(u zvef^11sNnG@lubMlBu_{g!eznA>r~5I3nZU)UDTwE-rJAAIlr695} zySv8~Fq6dTS@%Ni4Ic@yV}9nqWzjgn2b-$%fe)nm><)wj_EU%;acLpY%#W6A6%lCF z92-`wpgLR<)&8m;BY6~#J@I;6_`Mxt%!@dN8(=k`wn!5%A&*s3$FCd&q1TY;^l>!X zJEIu)UlztraJ$vg$uYq|1(05<%OD)3!=DKWz2_R&3fMFqQUAJX+7!h6~Chy z!+b_6T+cVT+>t-ysRcH9)gpn*?K`?<+oQ8KU5bWybSaMN`kxINDq%+JIhSw?2ZvS3gm%?YACr8 zmxMKnAhcVG9I*MlpuCLT=mZuBcUv=)E!6_*B{D}|OLJzsO_=@uTXnKg6iiu|gz25| zWa$4@cWD{!$5)R!Oru~bml&ekxd|gAm^HMtwqbiE$*Url%Jh9|-y`{diA!ZfSaDV4RvB* z+$5KJpG?I7s~LFy%OxucJiEoVn{q{cCY20N8oqRfXV)FWIVL;KMCej-#qipyl9ucY z`ynnf+cqVd32kJ`mnMSn+H;1UVKE6)55h0P90k^H@54D!@xL{}OQKJr9nIJsN4YShAy|uNPFew*}A|r8Xl9$k@Sq<(2P%p2{WpIq5zNs~450!lR9%2;`<$XcqB zY-qCP!j;GruAKHj{zNW&MU@z{^~lz=$EG)}LC?CmD_;A$W$l}xGB40ht^I!8bCu7;j9w+cjN0_0_l5Y4gHFhSTqcZKFD%S6yH=? zNd=RY%C%v>|}0WU%vlXZ_bnJuJk@x&h3pxR&qRO*b|^xSb?S2T}(V2LTV zl4sS5_m;-`sQvKk*4~wZBH!Vo^g7`m$|Ck_QU7_00XLoP=}F3RCI;$xNkP7@@f0`Y z;Jr6}yngyt&BwN*Sc7AD8tz(VsUqw{S)FLDs=r5-P%^D@t{EC}AM|jD-Pg zaX*lu-10EkSDL%f_&H9)vLN`|&L-ANHPspB zUW?Z=NbNZz6;J;#WEC4F@(da;_Hh?JcKi1w6zq*_8)TksSv1U#M#%Ck75&zX9t(!K zvxb}6Qg-$O#<(7!Zp9t9iJYXUgy=uCDAA0TRIzf%9M_c3>UT8t@b8HE6ZoH4O6UN| zFtumN?Ds~@6AdU!S-6d{?S{hi*M`@^xAh(uEE%YN9EJU6M&EvW`Eu+w>36Ik2>)(w zRfNY6+&uj6m9K(9iGQ~_2G(J#9K1!Vgw-a~mt5V(wc*-%t$DTn^x_pUzeZ{<-FCtu zLt#HySyzldu&2!`VYMqIw-lUG7j z1*@9^wLUY1!;xAGd4;v+)pM(#1ObmUVYbbbLD&zC4Z)8^*-P6t(IGA4aqOizElNZBLHKi7a472^Az`wy>C?Fk93L26VQM zMfS*po4@mPl<0Dd&wDJ@H!oeitesT@YRR8oyqc45jVfcsjHiMuc0AIo3brX z;kmxy|OL(Y9DWD zsP1Qzk{%9S0JR3_)u_(Vrp1=B^`Hz(J9m>hfA{L*<(pUYO?n84Vj|5Y7vT~Otv=fJ z6^lFe2d|TtslODKncO~>GMAZLb@yN%u@S$-VB{H{=^Z{?ttr=IG@5d4H>6&fljh5H zbT+|HzVHVY3fsFe&Dai|d$1}DH83NZ_th_2(er+yOudZpxf203bUFV_~ zFx-!~-`qgsUho$mtmx>81W$S9u(SI+nrSCVdOlfRg$Iil&I|{INkCpLpdF zs{H{pm#akaID!k0HmI#2m)9?@FPnamJ>#b%0;q!ua-UzMkD0=19!6G8)-=DU%JoaG zpa^5UfU@(ZmxFq_qce&gJSm%X;4g`CCDYRBHRJger|9nOSyEk*<_I>g@V^)N2 zAKCApN4GDpTNT=luf|rCNFi7<)HFBlNd6vA_W0jL%TkI<{l68>aQogqgqg3o{qeJF zyk;j_AH!Q`FR9c(v==P@}clRW6(d+;2XtP;Msg7dt=<9O|I zU_w)RU?(78b-YE(JnzoOpZ^v7jlAXxLUV@t(Zj=BnYOSXxm{_cljnb>X<9r-9=tS? zR56$^Zd)->;k?)<l(vrH7zxl9FT8-Afz?dh?mo**x1iJHrA%MAz_L}JpUy~_ueIPnCXH4 z4qvTN^v$DTL8uQy-yTHK=YQ{;>P$1c-oN>qysx1kbn%)O&JR^FX-(i4#w4MaZ?FIQ z`r?nTUcVuPv`6RY>E+A4FQzQYds&Pr($RdS6+vVE)=nbr*G4|AyLSzDuhnm_ z?+%6~VyCo@ynmP{nBWR9&!5A|1!POLp zg3-#mOOH)O>}>X?B!&DNlnRX_BKB!C-eZ%hEYqhVh2I9t6jtaDx5pMu8=7Mqc#_I{ zaZigh&@4JC%>fZ{s4QkP(V>0^g6qJitN+J>m!|f!sSxey+F@6TsGX9UjA)W!zT(-% z1d&NeGuzsdmG_K=rO78xo}lgV$yZ-}MW8`g$`Zb510nd-ub)1hlmB<|CKS?7OuJl# z6{~PYa3DTlbw>XE`kS|PomBJJAYfC8UgL<{&N+GMh2QzMK3c8^J1jxD5b%WRt&neJLPkQ`~jz-5SV@-@DB<)Gp*m;y32;h~un zD>$_Q|A_R1jacfkc(?h0$0ojKRD#jjH$|n&KUN~mA_~VgR(tef2Li_Sn$!&2&N{08 zus);rbEu44*6`lfEp2x{76H|m&h>qwg3zjzGblrEgRSS}2SK2&@)^`UjwxhrB&gl_ zc&G2zQjV#0`-A9!=Vq_3pK%l|Y*($=sS3JbXPVmD0>{m2fr_sLE6^Sj*EEW@dbYt9 z{sm=d&13YfSA?!s&|Ty%=9Gjc6dC`qu3@-lR})+#U@ZpE z33;Vt2^IY`WOG2u6$Xc3lG!4YNw}NOsSEf(cDOXq~V(3f$^ z6Iw#TJDZdv=;wVSj?i z9fnz9w?@dm3XA(>bVED9v+EK4C0q0fUj-3w_j0K*P3@pz4l?XE@)PurP(>K<+(?Mr zUOgA5#>%=W0X!6YOSPZkU15;W}KLl2uAyVfBE3t0R5w1f>$jCT?M%qN6tNZaz0X5&Ijttc|dc#{d#jgT6fMz>d*NY9Xb!`(Rsju z?bE08@j7)rR3|W9Osv>^z`r=bn8#55V>PdUx*Ay>o2;&b>N#?$yI{ zpFP;4kLRJCJoh~1N9*SKX#G4NxufS}^z@AH>Ny6a>HqlveLeTM7o%S|th;Ai+&GR- zo!27aURE=LrXy3O+Gv$Ms;Q^OMncwD(NOK@R zDHeaK@^}^B3*pzyboVe$)Tu^aaU4sX=^f=6U1V0zhfpW9@j>#j4X*0?SnRx&tI!}M z&>E;u;eW6`^H@UTCf8|yd~3{)O}rRC-%=V=&5x{*T}GcVs5az#t;bL}aXlJ%_6{_U zJL5irR!m;iu0qP_w%Za6r0D!xz)BzGPSuC?saWGY^Y#2;jAu8dPzqK#S(-xorXgj= z#3P~Gbk9oEO-dVf-iJT0c%d{Hpxgd>6c0oP`5T&#bSb&_{5}qZE@V~O>{$GJ#CAai z^io>y+XEY?N2n&vR8|N*XvV&r5k`}>7Qtp97C5bB`BE>|`O4aSXT+`%4_Uo{ z3FCKhF;>b18O1^b8B=%>z}5z5?wb7>)*ma!?RKcnxYE+4}c zJu2aSW6XX+UoA-CSIiVwrq-_KC8@7@5=*f-Hs&=DEhf0bK8r2tP!Gk*D#z(Ui2=P@ zrW7`0CL3DZveZddv8QyUH_w?T3d{5$3Xg6Z1vcXixySSQG4johqMT7MM%fk`SNILh z6YO;7mt=aYn6h0F?`p8U-^dplTFE7ko7kV6}%OxhELQ3%D8u)CY1~u$^T^ zg@m<7ydH+586E1?9W5G~*(i^&cO;;V<{t5!lQ)LpjYJ-*Wnd+r!3$BNrnzAldWk28CXLi|Il;WoJb;_7mw zHNktc$DZIl@!47wF43ak1u&79^|HPjBmP8d^!qDMJ5&2wv&uS-Bbsga2U<##vQRVM zx;bdPQ^@2G1EG0Dg$&>i6gz5&iHo50Kin12ZbsZ4}5M zOAMUVD2T(B7~i@w3Ut5{6P+)&3o|NzZnQ-zY>6RBlv}wTKQBv+C*D}D*|5rtH+1+F z-mGb+uDu&k*P=a)GshWaWYK7_b$mRZACEa5*So-|ZB@CP{gyD1$kgTlEqO6=l3GZw zZ2-PS3&^Ci^^CtUxV|w9+>5o^jO?!u7MN=>*;W0IsIt`_t|f_)m+Or2jBmKf2Git} zsgZt$69h0%^6Jb0n{U`g7TYriQ2%ixW25X` z-`*?4x)~K@L4s?l;?Ks+H6lt4Fq|~dOlM7O? zgy-CpNuc;UVx1{`97}I`saUo|sfdcYY;xf##YQC0Y$mziV?MzRhE*1{muBJ zV+lU4|Hhk5BWeh?*ezk)L}|`V@0rJ-Q|v>|3qba3mPnEAW4PLZQHxCA4vRLiq*dWk z)$eG=Q(kUeL>OK#t$PesX0L}uOjVybp?O|Nnyi`fW_~u}v`qqpBoNpO6g~RT*2N zz*AX%^W}(Z7_Y4CS=7K+P~hKAONp-BNf~~kJi!nQbFEo+U zluD8X*;HA{^UUo|a1rk;FvBH%?+Eow?-V=BU`yAW@-Xo|adTjzggyfhLC3twbP6zpHv54zrb6hOaCJlQ>Q>n-wp;XyR212{fYkT*l-NJXbRc~j5QTBmu@{FnZ z?Qg->X71E^u}II~y}Ee$=GA12kw`U^-FJ%$mj_BCFfm|?=S!R00$R&Bf^0fmwy6g1=uRkK>BvL_+aU$hX23-=Z zmd#RT&1(g};H~lyr&tVB)cZux=z&|Zw;K+IiM{A<@ay$o|7hDZ+-E&0SUHda(r&eG zUIG2wb2QSbTr*Mf1TDL2IN;DHUF33!T1CoI=zUZ0lqurqQ;mc{kb-9)V|QcSH!%*} zajx5s?UWtsj*W7(j-qpehPeHli=*_#V^A0y1QLGPA&@lP@PYj3Pl7}at7ITo_5Rf| z)qOQGmC%e`eiNmYnP;1d*|qu6>(4x(hV=Ca|BFK&B4pa{D46<#H#H&;>#=R|p-g*4 zOLkA=dA51pZXc|UrfI>HvJ}){NpOi5SC`~Z2FI-@JX;D4N_T+9FxNvVX<1%kT(z6C>nT}CMqg~RVE61q{1zrA+|7pZLs z>eY`kPYedDQm}gXML9Gb<0rDYX6CT1*tw>t1-auMjwA<8WiJY(?moJ=$q0_Ue|HSK|A~-y zVp#bJCd77`TUw5F{LQ$RfVSi^ud+}Onzzjd`ycT8hq$4<9GiF@+)vEu&hCn;eASW7 zWw9DGD6K7-PtCoAtsu;pLRWnWT0y|A!d4GatSYKFl}W0K4Lo`Bn=ikdlb5`J21DF+ z(5_t;v|u2NVwMx=q{C#v2=RfcR2|t=!+ywC6{VOviEy-y%;=8H)Q(T;h|0=-s9fw3VYhL4*U{P5BPvk_bF6#vrX#>UC^N?K z2rL=L7{-j*&N2HsR4cNX;~W8T3@?stKli{K8OK3c%;R#G$0Py7B>+TcCyyzn8O%xE z%`=XG*pZ1mm|YwJ))@juyqlF)HLvswPO;8r^7Y`{Hp8~HM;C{-(0txsP1PgXd54C zzY}%xoj!!`^dX~3e>yK-`t$VSsYY#0Ym5Wp0b-h5#B|62>g(>`#nmNA3I>_OG*j3V zx>e7{Y*+3~oG+s|I(V%|zzJ+*M?) zVEKcTwo357S1hJT8m+nV1Nuq;-X8md(n0IiCo+%P9m80S9l`CDz2Khs;Tv~`Cz@Cxo2ui? z95zH4nNw*Eb2I4kzXQ%4mo1n7$P(k7$~CY3E|_B0yaN6kP08`@8veAe+-RyCUB0KR9rk8<$>`gih2ss_m58!G4ujSMH(SATqS zsxbwuN0%?#9%x_M!VkPXJgu>{1*5p{&YO?04G)mHY5CVsYpM>;dDSjhfx!W`Hjc6+ zB!+w5ZPMGr7BUpFN8-ALFur9a=?T#=j>bi+zkpc^1Gh2d$yR2WTJD9Nrd zm)EBIrFjc4vN5jHG({E!6D^p^rBI<_J8{aUapqpLq$+s1MPfL@r3Z2#wtCfYW){%Q zkX2mDuxF~ZD)OyRtkjHjkx_9QX5*21p?)AKZwe}umK^L%_gL&TK$|JmEiBRDpb$*gN68d;u;#}YcL*R=wZQ+SKmmbmx*l>zSkp$z?pUUAe!h`E z@+_lsS**^Pn07xo}W`?Y2hha4&Z z*gef?;*KMHnrIM8nZU3eeb;2cS3_BIhn;O>VUK^PFPRu}9A4|Af^|R0(f`Nwb)EXN z*V+Ha_4Ull9;y$YX4(E{f5@RVXLR^BMOa|pA&;&4m6JN{lwls}3hWGvEVYt$oSlWuyp{-5J*}($+@1;ABn3 zDjcjrNg=aLs{&P?(y}%>mIYs}SP?QT47+9qj%Y(Q`x1x{JtVGIvft1Td{b?Rs5T2$ z=!Gl3sj`v^CM&HPIlw&&xFA2WLYjTsFlr0;djd!*nG(mPi3tL;CKrj^^rL0{+3tNC zjzigSq1W}CJPB-Re`rchSL}wDS&!|av7I-Vn&In0`gKz3_q*pma5KwHjn;|uVcxvH zZmUHBsccy3NLUf5veuY}X^~jZL%++HkSOG`Er$yEMyo+fzIsE0l@)iVhwpf;`dxWy z>!KVk7+TU|#loJ}#|?PAsUsb+?c9R~I>uGj|FpfCk5Vf?u1@}losO@M-_gLb8LU*J zHdeZiSk6AR@Z;;?KU5r9bZOds>*&X{;A_c+5Y-SSA)gMBXa;{dk^zOy<|MWcHADhI zIdTi>mIZ49;F!qHKR4Og>8=s6o}WFwpBUs2&`BcFA>lq+eT!q3;~6Ee!C`EOVUn>OrG}Vem@!5%#eEp! zhJG-I7fA@-7%#GCWU9-af!9|P)w*(0nA6?7zJB_}ki$HPCL1nBE&$!d54eD52Q2_u z6G{&+##;nmNKqCtYgK4OSp+GdiR%T{ejHlPMyk?Gr)U`#&q#y9AFI~GAzP6L21~OR z6{~Z&9Kf1IpAiZTOADsL!T*L614q~7($rjl4lllO^-P~_*l;;~jl)-X23EPAKRIUG z`S*wz_>MpJ{6UqptXjC!;vZSmEc209XOQg8d zQnVCmzDa+na+b`fNM|_nCb#JqGpLpN-Z6-c?YL!4;X;-)^CD*4`y`jC!gOyGe_Y4a z4HtT^wx&aHS1JLTCv1)sZ$$g2EYIXtqwy}XhGStufi8L~yhlUWk*1QW&;kb_1?#|x z&=h;+*EZ3@>G(-?nEm5@vS3&bQw^0*bs0|eB+tl2#)@(S$w38{nWaLicU8s;2f2$L z49Tb&RT!-f3%KEtxrDo~+2kN-C2W?gpE7}E{iF&l`Ti+Y%1*miUbVQRLC*rsP6s*G zhA9>3S=CtdpPL!g=2cr(7_4`o%&I}-zeC_-g_~5mz3f}@;7Tm@`x6tU=y%kjzS()L zjm7f~D``qgI$ybG`j69geRE4B_tf-wCN?^i69#h7eM zo<2JxiwZycevTM4uW7`8{?~kWZ-Suz_gQNRToGA;r_uyTLUUJ5waXR)_kwNE0=x7f z^bYl{9xhTjrNw!1v%RLipi z%=}$VmaKL_#gqQ_2~`?g!-efO>3cj4W4eCT%1 zj6mM{vGcMIU*+&6?!3Oc)1JY`{G#_Qycy5I9FIR>@Yx@Yx#?27E8`t0yZ){nvK?FS7^|52kKlmGsoU;N879%CeHT9oQMDN^`< zvpV;Ysri;}vIqFalP6D}eD&*J<9~mH|9$f8OZe}zFTVKV%ilctm#1I;=Br;n`SPn@ zfA!>Fo;>~PtFONN7xLr*zCAc9Rnp>Lo;+a%Px8QXs-(+h;O|dxxc`186ma+nX+fUy;nLqtvj_JpCE%SQDjc0KeeqmxlNMl`{OaFzY zk$O6-LRR^6()|#^v(u&}TR|Hl{NFRuvDxq!+?$`D>D5}N#U~{^Q`$R9>;f$-*!~KjxW|CkCjVWOuzlv)}QgaFY(>RcYWz3_+d0H z1U`$rvrp#08NBma-*tU|7oO)Y_`}KnZw}=j|Lxz%@jI4Ck?@SA$EW`_Zx~?hr|bj$ z`yBr{7s%K1HTUma;RMH3EJwJWa6>n4}bK?j83Il-74 zWm^bBgh2t?qg)C8ql4|5RJ~(xC`}hF9NTtsV%xTD+qP}nw(XqQHcxEZ&dvMWuj;G& zqo;T8y=T=_O;1&^SpAX=M4Mqb&xe4rnGEw>MlWjx;LHRdDH}vP+>T2yQiErd@?RR+0EPOyb2lMl* zEIgIRVEc^!x8ri+23+dxM*-Ou}O>Tk6ugUT^XC_b2Ml3PTe6Pg585!PzJLIZq<@)Y}J@;-E~iy9j>i3Upab|R%7Dzix? z#oEj`NoY-}+h#iuwCm_UpEX@|dTJvIfmEattd!fJUF-&m=gDy=K%u z^3A^NU2Qvci?}KH&iT<119lBnZZK079}QKf%5hE4DZ&<9wO6fR7SMrB zrj~hbzg*0)E@5Cos1t26UWv`XP*`H9HdGKE&BguoN?g0z8mTmOY%#O|&xxC~?5L+I zjXZ27rwOs%3W%&oF$V6@O_ z0`PYkRJaM32e5am#_x3^z-ob|nzX$8DcCP9>iY&LNO!SwCP}MEx8TO(D^x}lcj?bx zjQOAnl=eH(w9~(n=rw~DmWJYDohYynOubj@a0o*jw|WE<4}P>ZhHhg zmck`sUjJ1fNnZ8ZPrPOq65$3&o9#?iVXV~Ju?2Y&ilZxo5>JpYQr(vNga7VZu=AsQy5o}Z_ zW3&KYT4V$n%R;KOVd3X3+zg2jTFrB{IHNMRuWIIOozLqu^KPMC#@usa71K!}^|3*} za@|*EjZPtG;Sjtrz}Yq-I@gr*2B^Nv$m5`#8!4FqhEAC_GGXN732F^j8K*aqf;Q-C zhSi_^B|xKIM^&x<5&FtI4;n=aryvgyYmX4Ct+#1mh@`(pI~)R)!Y&}!$d6Ald^gId z)?23f;)C0a;B*%kCYjLLl7oT zT8F$iDDAK9|J6aw21vZWSMMju8t)>yna4*LR*tm=AuT1L`LKBvN1$ATPmazMvNoCh zdpIS}GIG346xzWo-Cf>P8`}xYF9H2!KN)uJIg&<0!Ue(eIWnWjq-5XKVVt)l+y^2e zE#!as3OWh)VV;nVJ&^; zxWz(_{09xx4u*6$oYL58)E0^ERVn|u<%#j?&>aT;R1(rsZIg1m{k_t4t!CS)`{vX1 zrp?-``nLU%M)Se7P}BZ8)06@%ZEWPnCyZ5-X_&X9(M~&(smAc)r|Z2h1&vr&vUGa_ z>H8%6vx57A!Q(k*74cU!CvXHGtk!F`s2#x^BIuHb4oJc3XRjxr#)Iv0I#L^|MlvJD z8Fp`f3==LpRb?O;g=u$O{b=fA9U@*kgK636L2fzEu)fi3wO;8mH~0HAZ=$|<6R(bo zR2h4kdBmw`V4&_eLjB~`mGK>fr#`$}rMyeb@w>{I|;SKsRjHNztt|P5a zFd(d18Y}cku-B1j-qs?K*!0twtKI^`kqKo@WpcSN23FE71YDs}q*)R0(j~-xf_l%h97unFTqainNP-&fnD^&W1&#cwN&DQBzZ=pH=2HeGE8R+N% z7Ck!@2}AB^%%d~KW;g-b(Usg@A2Hs{2N81@7O&(^w95iy1PDY(>q%m;grr6>YbS0&mfAKJHihH)f-G!k6 zL``tot}05w+MhF7b>!iONPnE}v=5pT2U}Sg`Q2^U87G^yB_Zlz1#y|?l9`{@7!Y!b zEWEQgypiT<_E(8X6Pv&(V|;mq4?G`s7ml%>LScDe?3SS?rmZF@=M&3Sn$ySqx9LlB zeibja9rK($Cdu-hJ>nkZ>6GPtVkN;Gjt~Qb{x%WX-#ajsZk#UxmT8%TfFjLtP`Xl;b`jZS9T}SUXwLcOE?J1_Xm03eu9|BjYz+{@T1ru0bktTdMofXZ z|AP;>HpsPhT5PI({?sxas&U8av2VF#L}hxR>Nqnn8<@p#szFvMH8C5NNmxW8Rl69M zs<9Jc{bLO#?O0iiqsIG2GpMiaK?m%ugKK(p#?8XHXl;}LNb`1wO4T}|%SfbVt0VDm zL6#Q$u7S3}9g2?vKYbW6rGm-7jI(n_poGhMHP~lmaq-BQ1cgy7lxK&Agy-1S0DVjY zE7!oSSTt|0Da_2Cm~y*q9!R^9uw~Or;$ml7RX*%!w);ar^4)Fb+zVRJ09}pn!dPA2@uI`3&W2MJ z-krso&XFM);>U0?iGo)E_-J_v$i7!b9r_O-ow~grHe#(&+LnmrR6Ib3zo}P zGL1#bunl=`LurC)JAaT78_clc3WL4sSRv)dQz11ID6(sU`jg} z?|bcbU50sJY*yKVSV27uog0vIYF&jHG)pqm^=(EzH2pd2i5CU8Qf!rTd*Knk0yfjE zdw~tH7Cu?&Ba3M)$&xm%WYq=&spHq_q2^elL_-mOrcO-Rnu0uV7^A`^c)=PM`Xzl~ zH?VEt#@2yS>Bhjbo{#t_q^1bN-=%>#E=qJi2n&T=f#o9$K*7EnhsRlWHmbd+!74Hn za}DJl10G9m=3L9_3*FU#XI_BW5eHOps#q1+BXeDJ zBplRhkNce;r*mxu=E<%InVIr&Ocsu!(9y%_Kn9K?QnRJQ=*(;dSLy#RkZI&7%CmV8 z#!+O+V%D5^X_c*Or>e4$NOrfLT_=CgJAmbs9lE(x3C?^>9^b>d8P4PS-dN@~dNyLi zHT(&fY*c1=!w-KkofqR#&fKE&jZ~J`ELbQb8LBx+Db-@U-^i}$Xc$vbz<=bK9vch` zPZIRhTH^-YB*b%F^rdTd5zzds6*xX!$f6_j=RaL(lp4R(!^AxFL$H%@K`)76gz>flyHTHD~m}7e5&54?!O(2VwzC8PYWb zSb&+4%vsTWD;|;Ag~NTz^Z4?6xd{UP1PSW^E}9+mTCcEtua{LNVsqJS{i*<}3Uo6b zI7b<;Z$-Yp&^dA~CP;d7-mgM5I)EpE+rnUHBH^a^v6WZ5IJhKWjIMj{#~4SjXdg^_ zZ0Q6$gPbo`ii0SZ=d;^=GuH(aC%^=R6#)^OCSuBT){=yl=o)NE@R1mjWew zOq4W!jsQr(E@JDe7^IkAvizfh!D;>;Odw7Q1IjRXQBz3Q7A^=DvfHmZ4lhIp-7zQn**DBU7{LSh>^mkp$93TV_#UtFl6x@$b70O$6fn4OMBL1! zM{;%U=+x=kro^c%+N`h$A28*5N(>W36P1=3IkFkTDQyjIX-N=2mxWDuDxBaa=G^-H zKlZq!2qsGCQFjZlfDzN(r7ljxT!(*b(^Hw`OR!+*SEAss12CprIEe~O9Wa;a+F-y& zj+iiU<0kYSFr;byhm5BfNt5|3q-g?H(zHQ-SZM;@wWGfp455^0WvtxxD!~Df5IP;$ z3Y1D}wqgW!MhK>D=Pb!h3=pop*|B*u`SgTg2M+gaqQ|yFB0h;TW3%$xU|<@e>3Bv? zte^aodomaktL9;vqS_hZ+A%rY(E{~h#}>kd_S#td+O{J>LoZ^?ZHOd0z5Ow9cRV__ zp2xB`#Bvh4DWyK6;j(Nz2!119c37ix|3Jt9nhgwa7JqxChnxz?8GhvK9;GbKy~n(z zB}lXL;HFW<@g+zy=F)v5Tq8qOJDN?@u0*xi8J*P@2mQY;+#MgxY?gv!uLhOKH#cYvL*e zbe=Mxwnt(A8UdqaKyF+{|2KlKB$qAstGu@~z1MVcO#d|W3j;E;gdFyZMM}x}-}u)& zXZX)rW9Wd{=a;1o`u`^k#Q!GG?du`q8fMa@z9Z&iTm{+867uN(fd8|~@bc8;vitBY zNx>5OD*bebNxsZ!yDBOE90?M zYc)zPWAyF=|J?FcimPS*dnxx4NJM8SSc*beU9(=3>E#%PFk}JYrl(3ioxJdi(AK4} zu0SM!ZvVU$JfJ3GKG|sy1Wgjw{4mUIOZ&_47F8uhjQZ38YG+!K?%*G>RBb6sE0ri; zAn`H3wMZmW5qF&n>Y=gpRI1XE3i^`}s${5H#q;-8k7xqtO%ZK+L%&My3)On@8vtXOOIZ&eqVH(@s4G7?SsHxBmO0_?z+dDj}M9)!-`T0DwR>e zk+YhC*Wb6}BevObk<82$%M`;&N+zACW4sJ5qBiYfhY921MLrz9$J|Mr8t9^zFRa7S znp6yf2+xn{rv*XNib5Nv6+zR(u~Ssq;;FU#3_jj17xx}~5vow|(YX#HJzL9#KLDXv z*4YrYjLxRo-SoSBg&!w(yDTU42!bf=Av`!w%I3VxqM zh3wa8NmpdhH`qu+A3jc7KvEu)BDdSj5qDlM>mQ*urcr=mmwFIC=SLg zHb%xR-M?)b^*gO`T@eY<3Axog+smeoc?J;ILVFg&VpIG;(5To!h_8XfZ3KGs-9iQo zq`x90Fd6KDeZRpj-V&L#X+h)@dgGN#XO#4ruF^CkS z2-9LhS8`^$olK1yM%-U5gf*!i)k=3fALgA-tD&ol(~FCXe-HJ`jLy)4!YvHYd8yax z*=;fQ@U7*l@nF-l%AXhMhe0RG`Yj7<0yl+ZCU|u~?@t|o=@V155R{S~Q1Pwb7(MOd zPV-uo&`r}DB7GdF8XuNOlct$k!y%PAUNy&ll!zTv?U!y7EMrtAmIK2>(np|*g`>8F zNvN3ie`$qY}`*105hVS-JBTnnvySeRVJ zj@!{EiP*23?}y9#W$*rausFWs_x)(@^eFQ5Q1jw+{?&EwJJX$4^5f<4I6O}VuQIPf zI?F#DyEDD2HrBOI4K6fQHE-bRj^M<-NH1_kMw_Z&1Ctgeq=hVio`n%f5TTA7kchRb zi`*C^a{x;Vrj*U}QhHM*)iAKlAdHA3P8EQCOKyV$Zj8{zcoiuFZ$-jHfJ&b_e=W^0 zW=#}OxD!ZGBoNJkMNEHdejI)6q$`xp9BHuhq=GfqG*VE?VVrvZ{4RE-jm^xFx-D8%pSwekOHdR{q35`&)GVSAjTL{N z9+d=;KYl!KHZ~>43Ur2!>jxuSSC^>vGDJ4bsKMoTLvxpM zx6x)r$f%8yOw&8%?zlJr*jkC%(qp2OTFh%Ff}C9td;-c^61#4-q9&L5_p_9c)&vjP zPVP(7*gZnEJ?5SvcFYhECj@#&_ha@)V74sp%jbQmmH7DSbAFRLzgx9g11;IQIU$=S z0?g+HAAM;vzZM{o^o2b~N82N=lN%m6ww*yG_=$({X+f$*&@+PDW&?KEMi5xL)o%Bj zO0YO4Y)OydE`Dxmb%TZyF|~0@D5TahIQy$s4s}pcNUtMK~ChQ^UxCP}0@V7^q{|;0eZD&_3}t zu_7rI=8B)=ul-Zq+F!kCnY#rOPV-S8OW$t3Q0k1UrHokajCv3Zm4%Dk;jVmjHF%31 zDa<2f!UL*d~Rg+eO`Q``ZB%= z^O_wL3Z8PKZ-@pu16`1VF2vm_cxEc_Tr0%5mToR&;~<}K z@eV)k5F)-nUN@P#%!s*uwv11B_qhGsGm!y#lSy90-yDQ2)Ik=W%m%lp;BC!wH^}8f zR~Vp@3;uoWk$;fO#{wPw9*oUU$oYITk%{ENm}>FJR?qP`co4(ch?^|)I8+$Yk*A70 zlVK`H|DA+;SA3XAf(GyJZi<;XFD3Fa`m%Bs1yQ}_a26dnCO+rIJl2bk_MqVX*8k9Z zF}V)sPyvdXuA*A>-uEPN@6guihAg@9?j?j-FTyFJN`k)&WvXmv7iG#_&-*iMNcx)X zjE{#_@8pzOibd$g#{1{#(jQ1_2-XVFk(y000q0HxTKsEHK!G7YD05bNy6bKw8=jc3 z^+OJ~dWm&46^@Y9d^cV?O(_^Iv?SKQ>T9=L=5v^Q6jO$r&Sdz7LFm1g7Cp=kdy?AO zxF5gy2M@C6Gf^kg- zg)jwGNK#fskA&d5J}eCuMc`AAx6C3dX=hHo!ROO^n`z*72Zra%w*2Xq*4k`{(g5e0 z9bd&ohp`j&xJYao4Rh)E%V#}UjW9k=-$J?X%Wj{IqExdMyg_Dec~fu4a2UTMr%W(FcG5MqRccUi6`&ZC*5KhKLJLE zO%z1=IS2mt#TmF#=-VuxtRL_DfU+?!_vi6o7QCKM`?vkKfva#Y`}_W6>iZNAmr{nT zP;o6IWkxV-vO-0(`~FYv@;t52c#gX5jz$o}XlqQ0wgK{3;srEJ05#WY7wBcQmBb+q z8PzOM=}Bg3s7?Sze*Hq?Bx-}1l#FbNa69N>rkTmLCr1~XS4qY>-#Px%&iC7PF^0QW z0p#BtUubRhIY@*uN(nBHOL$UztfL0<1L}78}+d z282)(1EM?l0-0(JL>3slCac!=tVhxLEW;31*?p1J7KTX?(A9~k!nrz~)`QP2USbwV z6@QSVX3H{p(GEf1nLPCl9=yKff}W8D7-N+P*W1tY=Dk`5q9-GYDJ1IM>&zZqWrFkV z)3;YEQq56Ni8lyzDgR9?n+!nDwc4t&n$nFn2LcDUu%3+)PuW&;N!_zHm<&qA%Rzxw zq?2;a?n_a~$FiBzLi7lz2W_O!@z-SBDgGHjm01tO=c==B^1)hQ6aYia&Ja4m64U4P z^leM0$nx2oIDK*^@UP#k=~fZ6G?Rif1G|(AhY-38`BBMI3{a3Rgt*? zAU>^Zfi(^~sO*;^TPudbL>0cbzWRH(MCo7hos3%2Sd3-gh+ItSGc{M~9I1P+SofSH zrw z!dx|3P6%Xn8Ub$-IrG#eagM7^Cg0u^T^%vkxfgqd0ogc2INf{=u-e^<00JMmsZimM zOkZ27TW+UOoQX23X@-*eOI)(=mxx5ob$5Fj63rNA4ihSTS)rE5eJq*{qk@bg&m^!dkB@lm6)7ap|S+NvScG{ zP6mJadH=oz-_(Ylwklvb2g%7#VpuM~M(Uy%LC0Yyj8lT9u#L5?3I0(%Az(cx1Rd#w7xUEm)(6(+A@6Tu9iSJ7s9CRe7k6)0kGqS* zw=JD5ay>QNT^3BiM!?aMayNqTs@W$=tV3tVMR6QL)cm%x(UGjjv*53RoCLmv-*4qi&q2D%9+cZKE8bK8UedKOku{44!&l zjDt>zW?~Isieq=afd23deM4($hfDsZtNc*Adon}?UhvQXSEIs`p0b3rkES%+sT#DMdY2&Thb~h-t4cak zl`F1L$qLt!BpIodEJC1NnDI!q@qrhHiz>L>&38I*WpPQ;S^V%RwzXp*84YED+$Dt7jb}Iak&&=BjIN1$tD#> zcN{;TWglu=$8PaSG!S;Ef|4pa1f(6dzw=Y?o_hfQWf<3a{i6e}!6&Q;MMGwi^%rAn z+d|2Okm0Cu+8oc-WQM-cAmk&T;Fp;ljT%u&371da*rH0VnX}tzM8F|=mMMuB5-Lu0 zV@E^_82DjLDwshy8LsR0%>~R^*9>8^v?6*~vLN{t7==USbwk*ew~4A9cy4vGiDu`% zqO&lEuj_T6$NNQJp%sPy&CJqIBc_MrWB>E|czSw!J8HM>?ZU(LlY!^+nvn7q^R^l` zh(G=|o4_9%+jDBAsT&7oQYhs^)_wh;yF#oy*oVhghQ&|MM*wdQ)0i1gIGV5hZkHFgTXeEthz~g_ zWNxvQ=$s0Xw~u$wu?l}K;ytn`5ZxqHC9JJ@(m4rN&u~yYz5*9WZYBb4~6j}H95-HQa;o`U!=G_5@VMIlJ!5y ztNVxbgKHkF0r9=Th(VPsT6#q=yU%{|Ad&FVs@~g;RT&70f@U7<>1bLZ(k>8kJ`A0U zpxmxATf0xY*!!i>?vY*uAN$;X2 zI9ZoS{Y}+1n7P=P-G5F=C>VRjZBVq&$uwQOrUTpInKDx}SPw8|IAaK>7Xz7vl#?|= zq1%aYl?SQ{j=bNqU&N#1^O6FJj1JZkX zFxgND-$5i1j>?8Q!O**54r*22KVp#s!v3j?Bwwg6hxQ;oM{i zOgs&3?kAcU^oXIbyu(+Gmr3a1!()Rt$&iRc5l~gE_DWOS9JPQjLw*_tjt>`$s4$NB z_vn(p-&J2>KC) zeteEOm}KHKGrJmMW@&Z{yVbnjA zuRbZB2aa5`EXq0+q>$XxSsuvI!wpah6>k}PsEonZ3E|)6pfKhoW-quSIr7BZq-;$( zzyVdNP_@!wWOxf&F}=!S@A^&gYPV9uC*^^*#rTkpdI7Q69T#h6Dv%jU_2Ov5}jZ zkbFi-trGYgBAsQ(00T=_PC>gdj_RK^I3PCv#2S zZKhu0zX@E`x0slK#d}$%MaiP4+&hlca%lj8XA-xG=y?9C&nuZ_JL^tqrB%K)Og}snMt7NAk&aI3MSiYWsj5+0~(y#5S@%S99Qi<&1m}7pHR>-0(i*PkcX#r zd16FbICqhwtet;-{$xu=Qsr*KD7&0b*ef~vureV$N@(|V*AC_fc?Gvjy{-QX>uFtK zsfpN*7gQOKa?0M^G)xFwhN9vu!-OrtAyi5fL(fhVU|3Wg|u93WLyz+EIVA z4zK__o=?nZ6hNR(L`lh*V-x*3e&6obI%zyt{vuos&%E@k)RRu%z?aUf2N@6Ej9U;M zeWiJ|UNQXzFJfL|A@+cU5)8{w?xDlGRRnQ#J8HQ0T%w~!B&<)@x&cbVyR-L&X z4NoT9H6w>EssrYs0_9ktBIBq&XQy4)ri-9|08)rx*!EP8cTlkzPxULrukCXb$!~=2 zD3ZkLW6H(DS=uO~cvsvF&JTo`_~?KmCMx|*E#S~lB;9IAPfR`j>=O_)Q30y>LF%$X zL8$M*g{-sd+6p@Wya0O!I?>pt-dIZC9UxS>V0|2TY=gi}g)z-egt3zj-VJ1|H-_+d z9HS-k$-Zs#?Kc+TjKk=H1tF^n>o60;SK$%GUp_6u@=-2|vwT#D;jOG;crX9;&7A)i zxF7v5h@YTh@u=`-^r+AhJuk-cU&d*XWh`Gh^KZKEC~xI1(abzB4g3LPtEs+zS;Q0M z8&5E$z3yc34CvAkns#o~)8|r8=}L6Lw{x;PQ?FE35*aAlj~ezH&qYrIkTYT^Mj$%l z#m#InzLYPK%v+qn2)_A1T=F~N^h=KgZ|F^^^oy41TYWKtP3w#Fy+Anw=|?FCZl z(^o46uc-7n)ADH69#KsQaJz0cCQ&4{7W&(hBtpl&#tp93W*uAdeAg7qUrC>o)4%s^ zlaiqzhD&siLAXl51r!1IG3lnnEg3OXU7@n(CW#0@UoF|+m_YCUnllYlHoi1=Bs@Gv z%w~f+v5?GC3&YC_^t!WhJeX0K!~`j{9$;Jnvci~|w!sBr5FavKX=wOaXqctaTPh+h zgNq0f+aTq7^icp1yu{7p3C+MXBQkPd zeT(hfpc6R`zv30W9RcJW*!vM>yQ!zP=hZPNI=aK)AAEg94SGktR8M!1YvB^@i)h@1YTSZp+<<)mDi9E( z*suv)w5-D-aL*4?yTUiF48MJazJEsZ(Y0R|eD}o3`UcT{vU^D|c6C5;b+<~6U$wVI z>#j$`>F)W~o?eEm)SN`erWOQN4NWn&HjzbEQ+A^#{aNpHv=+<_!y8%sM7s+iguzuj zo;espF13mh;!D&_Ua{l#ItTb_JqDt;7w^EQiUxhAXNm@GByTFE7)dwVzoMT#PfVHB z=S6)sN+ryyZD@JxruW+lif~+F9WI|>z-~O%&U>PQ`(S*AQAHriPED!BFdYrTN&Ok# zmkBnyhbQ0u^bGV*^L$9@gUr~9g6^nN0RLE;Di?GX6(>*?-if3Hg+dBIGfB0H{{APf z%fx|sEB+HwejkoiAijNRIbVXXX&VOo-j^)rf3&Yditj`0ljHY z4upWNUHp`LBOi69^6wqSR`8?8!yCn4!ijV`Sm;@Ht%1OXH;fJ{@xnb)%>HWXfrq6} z%R#BVV~!e_VwYa^OkA;6mot&Rb$tRBFQ1EZx96}hcP9rDwGy?UUIcF$B_kgfF^C!% zuK>ZCuShid8PaHKhH$i0ajb0S9r|nfRKcgOc$}#yQLwYj%-F#9U|b|z##r`X#+2wv znXzzMwGu&!UK6pEcOjAib6J#Upcv|egm8N9S;1gzL+Dy9cxW03)e-{p2vkh!mh|o$ zF<{E7+#!;rVl{wp*qA637js$nisw?I47g1f|KQM_IQ*_nD0&}LfmW!KpQlL0_^PLW zvRqoXUVq9|)Fi_pygfXgCNCE0X7${PHaTQfeSOqJ>E}@LQK`B}A&L^`{nA$OUDf+9 z9r#36*I*@e=Yn13576BQN3iTA@*TfogNHj!pqfIkiDrYS!TaOrc5?Lu}i9`G4d>ICkK?lZMyZ&L=MYAlAFh)_pT)=#Pa{Xj#$Czvb}S6-?X zike%frk%gJ<>b@S-0rY^4Oc_g{zH_~NPK_zZ?Tu>M=B2IlUzo>GM~{t&ue~v?W&4H z&nLk%Jfzq<U@Rw491Zt26qlXX-%gU^9f=i)&| zk0@WjTLpgpNDr&N6(qdO1gbksGCAOvvoG(d7j1LHFw~Sw5JTac6ok$8MlUvWSDcSw z&))u|!v}_l?0ur@{D+8O{F%Tj`%kOdl~P_DnJqULc1M|sAOx(}xoBv`m56$U$wI+F z5iDBs{#33%W2nzwtg_dJ^W7I(6P|PAiJW*>Dcj`koGh0*a?~~BiQiHHhTt!40y8PF zNc6Hh@x*H+cDC4U5XcbhJMHnAeGDSo*@E`N$av;1h8h=9`8l;`yG`{5TOk5GUh+FV z*|=8vbHjV*)R$6sB70=3W=;HIkLAmw&8JqkVYZ}Sx3m;msYEne{AsD!jc3OC4vSgJ zVK`p~O#g7&Hb6o8DYlFbJE9P)cAvvZxdJl@IP{X1#4X^E3@rL9QfWWiA7#tG?#_+0 z-!am7M@r%x&i-F>6m6Axq{MC3K3WR@V8LzVzpR%S23ntJsWzkkoSk`?2MZtm-?oD^ zBkhk{4F92U{l;!Ed-?86qowSy|6k1iFxt2m2bVz-9{G$%m@=&k2~D>DrD`wqL_)c@ zEg%Ob)V)AW_?*Zr!q2$C4qO_)6K@49MyJGV!O(_*LgKwTd@oa?)XC>gv|pQ0j-lHm`JZSC{FUykFUiSjesc2Zhtv{Ep}9$dUo4jX$? z3X7z;U7;x>Qi07|B2l}kZhxf`IsL6L`=j2SwgvL+0Q7dr6?wvZMu>d8GAgo|bJsP%e_srXDyJX8q33gdqlddNvg}%8V1ryf?4{-+giHu1CtvDlCXXU z-!HPf6q~cysYgiPW^%5}OAQeP>-VUJZ$30eF}$-qOj^gO6AA<+zwgSHdUw~AGDtJwxN9l{=-jfr3VGU=2a&E(riF2M=ArAl@|r86O=5%-k7zqCxXM0U2V&T$ zso1om`5{YS;||A_?3EB{#h7To450(>02JF07+W8Zntn;RKHXd|oZtK7;NfZ1;2?gU z?DzLWSs0(pUKk&~kHcgC*TG^@(QlyF<6-&f^3%-oaZgP3nC(~(AI^{Xgij2JgY$hq zY1@LQPjlsPscx^iiyq!5%*h)(6KTd{JQ*?+JQS}TGxncQ`Gnm-t-|GSV*?ve?zimi z(g?$~e&zDnO;kl}X3(*`Pmtu0Q)TmYu?S>b!eiD#t23-u8^JIz)#6gexu#|n`PVay z*-#U1yM?at917<3*ieXQU?f|O>Krs9gCR?Y6okYTG2eC-mQg$!7M=cY88xfPKyUveDajR!;x2Vuy?Yt&0^MjX9bXwNSa_ zg<(OVut&BLk=W_9&XhAxQC8@25bIZoQhDHm2()`->fE95x3Ub)J4XvxE3Gmz)K|F2 zfszMh7W}hdDwci|NCsC@*{QFNVdX1LteB6D?0}eqvnC$P^UDxT2FJ2iQx*|)(y6!h zH{sv9B*UJuC@1S1TY0%&`U*G;#;^cWthB_y*#zbxCFqceHkRC%%z|JgcoK<<@+}+( z(g~fiWWGt~dMtXDb3Wh*cLRroPj3WnoPaZs#zSY=Ac4We6F89aD)KRHyxWi)9lOzO zcRM6H2hsFL0+|6tqu|eTts+p9S(!yi68(#r0eS5;0eT5sPXtPs^>rEfR+$MT9BIEB z1_Q(0P;*@Q{gn{8lQJtT3M%yE-LBQ^sc0BXN}VP40Hh8cmhZI!LU;jBdapvDm+WRd zOqDnRx30RZE9nk;=^a0!z7vI`VCV+>k5{T&wvMY^Dco!%$MVWW?1KZwR^eQJ_Ivqf zV*%P4gi$|ziw+-%+V9QEn)GUY+ux5KkFuhqoX1feYFMc(h$=2SeHat)QjGK`j!-UD z5sGFNeW|>3Fhdhhf&cKNz#wVsn`dCi-?|!ZeW3?1l$=RWx*L#Q2I6xo8Q)Oq*|NYn zcPQFFM;(u*0x^YyU$w2X%q~6~WP`;P?P$m<|2RCp?HVqhI;-bCTVy6SH}L&Qo99V1 zoWXIQ1&lDYtE2tiiKioqO8#IHA2Gu+pTcRbyRp6sbTGruCDw6BeeK(HBe7DouQJHb z{{1&$svw{~&E^BBg;1<#78y9PKaL#`k4erkS1x>rTn9R144x9vAg%7|rbme^_P}!E zJt&2SpQ6gcjcYi7BsGFO!O*xQbHfrgDbTmnO1uCKL-PUYu$xrDnC2BceMYxPQP zqi@aynS?7v4^sEpF&`K3qADs4ORgCN3JOul5IDujRnj=ryGm64TXiSChD%{|ynxL; z$ez?ATH11}YojuK`R6h3M7yRCmCyJ}uaiEm+kA?XJ+x6EAN5rkS^y~#DHX(|4~pbH zq<`fp?I_#W z6a2T>%*Z)0#+tNK7coQ?ln|;|@VMey(7lbcYvhh@*Cps13hp@v9-yrNpX+dpkEzzG z7W-{~WuUzV0($5z4$(*gbyCU>|28I=jiw6un&ri|$aalx{KiTypc|F*Rhz)gX3dxU zq#X%jQ20@s7insbc$q_&gbz&PvG5*B)pwR5g!S^NU!McG$~+w7hTCR0{h)DcP%_xk zEkefEl|%(`9R<~mwBrOl)7UacS0qxKkbNU55{t`9^q78j6j&{eTX++{UOgle5g;cG zy_81}lLQS!l4!!EMG(Sp{x&%PcP=71krfj8M~4oDc5i{%UMv=WKFq9ysS%-!z4Mwd zJ%mY^aNM1>I@&i^E@D#LK3a)@x3X0rcv!VWRm3z)0MZvjO$E&Db`Fb$>6SLK{ZyQp zjDjeoL}KvCu(W-RCEPyX^YVIt>A5Hj3@_OFKak*%wBXQ`Kb5q|S3klN%>0n884(|X zgD0g0>wJ%LEQ#K>Nn^fDGkCue==NYPqk~})t?QM=x>5vtqLdM$Z|{i8@DWi%K)@Kuhy9x9WH zkbP`WOr*}LsMvSoUTU6g-XjMmDw!s0rSbKFeev9SdFM-*K(+>6=AHdqLeni2Eb%Yx zGo-vp9~n`G|EbLD<>Fi=ulhK-tb5}&<7S-*?%qI@ailP!xE74kWYn?u3WRmnIFKgN z9z}>eN9YQ))3cgrpe})A(kiJ~!bvk$r~-l;c!~#C43`GZZGtVb3F1F(vrk_ugg99R zjk;K%F4b^$_J_1xXC&CXsA(@jXhJ3oMn?JS^oA)K8VW-WVeaK%v|#w|R&SpINIt92 zf%gO)GEY9v7(iUYUNGj?bzE~RqSiAfv}z9At0DzkoURz?L~n-(Q(bdr9X`s=IzlYf z)GLkr8`{}^pA}Pb?yp>m_hpQVzFGu746IEq4fg%PS;NmBx9>KSl4?<=oDipm!OTvu zbH?-D>U*`;Tk;-a_@`>*Z12N0eWMAx@Saerz$uDofmI_EdWI2hL)r(f4lsY938;`o za;wHf4}yOO97Yu6`6X}k_hK#U&Y29T!wIo<#z;l75BUP4O)^Eze*XDK95~=><=Y@I zH;btv3UK);^ROYyJ?`_ub5i+k^#}T}2#V1K_1+X!-H=I zWhMr6hNUTC%=RvungWE=HL#X_Bp?mT-nv+zQeHNVipZJ}1~o0-$=4Le?FML5pLUK0 zt>f68T_fNHl_@zD$di{maB*6gvbtGbR=QP@tw+dQiqZ97$By;fnu*zEBl6~a!=&u0 zl2KYt!{p4hlTjM3U9^s?{%>0!Su-i~3jM#48H~=$U*G$$@AoXD(f_)|uc_Y@_|uzK z_s3s{+i$`MEf<&3W#b}~uiyUvVeg)TED5@XQMavWa~jjOZQHhO+qP}nwr$(ywB7yp z`+f&;aCPoZ#N1Tw%8FePJEC?~R_1!vn)*3t^*??8bCUmliNSfz%&nNEnye^`=~`br zXm$6S{QzZZVkjS6gpn_><{E@zQK`4tOH>d?fGPgEA;k9yu9ley&3p`!Ag>lo;KW8p zDRsW;wF8s5>wN=l0K&n2qFpNhCu1`?RqvU7to4#+O_!NKO>a0F=GI>| z1XDP^-bffh8H%LBTK<;n2Q$M~{}H;fDMSQ0F)jcdqhre9W&PDjGRQ>3AQW+aAc_bf z&%#;1&KfFMHXt4|rSld&NgdO0)t)~hd(KX?I4n54CZEbro@ki(uzE5DBSppa;;rO* zvC5J8?V?Wqx-M=Q-jvEYIEsACi#|Wg(e}198KRmksB&aK;9zzwE)vN#XVhyxhGFO~ zyIzajbdlOd@N&AXNl+r_#$1GKP(YcHVoxaMV{o!C-&c*M{;Gly(^XJ%xq6Ta_3*$4 zrgmNlh{AfHrxTg)FBQfe)9ZCeIq0ZxicF=9)L+euCLo1i=OQfPA{mgp7cpP3u1_Dv z+sHW5^7Y)e(^m1fM~y|#3tbVQExD$iRjw)&5LGW#WzWn0dV4T^KdYC2K@FDC2^kh&=`=?g&eR}iQ*9=y z3Xe)-)^FlRD+}W<^X^t!rS==a>6r!-g6?Ow4s(am%jcv5f-@_nyk9OIVYNcKT~#wV z-d`*ZQV3|LHZu6Xm40LSgD@&FOk$g)E!X=CD-Z4MZXmpGyH@SXvZ@z$c7O3rL~SX` ziN#YzO9cxSmEiy&vDYrb%F8=PENo@;RJti29}=l8tR6c=v1BF@6(E-5O)+F|<}QB8 zJubVIWt3dCRTS|I2%m~XNBGwlg{7dqEK9bi^Z=t{?%=z)*=SM0?s6+{*H^z0wOoOI zSOPVGFXi$(fr&H;4;VcB?t|SDcppy9O&*3)lfhz&gd!kJN>B6wCxMBY1DdFma)#HD zmSh=!i*)~e4tu;R04?qSm}_XdnU#IJ*bS1hoTz5?Ykq(KZ-Lm~Scy-quHxgY<(@L? zCb#mF(r2F}Js@Bk`b|BKB;pV&h_j1XbA;7T%`G4bCmE|PC@(NnT6R~v+LU)oR0>zW zl)i2t4U28>C7H6Ne%ue7$sIpXG*mFL3m^d5_9te*YB8EKF!V zqiVMXrZb6Y(Ip8wfTo`k+AwvAc_s@cGCt7rPz`GYRVgTDOwYzN_x4Sm;FDKTe3~_i zG)_a@dn)RXgiaOJz1_I}qJ>4`G|vyERsPO{_5)#E^_e_>pE4}0W81nvelNjNu6FP6 zLwpeNxcS3PC4dvJkt0Chx@|hYp126*=B_B$Yx*@&+YqL9cR_^HNqsGLq3Oh%q`A6-;fup8{A@%piwt3cEIG|gW%IDWwNPtPUH&sjDnWslh^&tgT2 z=NhiV_x&I!SKf{qawgiaXKA}_%bOAjT-W>Yg#jD&pkADP$m={(4&%}$Xntj!Y#}8@ z&@OfH%`8P4^dbH%Fj>>oxgO8YB{$SG)IH`KYYshXu8+x&cZgHZC}8+$waDRuqXwU2 ziM%z9$P}BwarR2(i97T*>ccv*4t&L5B4rPx*5yWoQu7+T#Oar;bCr%vI15ZoYPYOX zUCp%+Dds|{`NhRp9|_QJIn~<|7U5D$bsbSE>sTOA2U)Qw5E+3wBNE~%)my?QNRzVT zafh}DHLbK;m81;X(ya0)T2$pz<((CeHPZBzp^7Bm)M2Z#&76IpT(1f7XP!VK%zdC% zXW;0=9V){(JNadeds!jQzsFy<{v0BEwb?0_z_Ovc%xvhxR?iH|R1Qa;Lkg5WI9%H| z`DWUDd||(Bf{rnVrj(C!ra-b>@mgoS5{xvKaQ%suP~%4i-Oc&}3z1Z}wy%sznW(5R za(EN>yHnn#MFZI}<$`U9g5A$EACi!5#RF~x7K+X3L=Pfze)skRW?K`Akp65-5p_XQ zbO^o?okn@pp4hyJ)TjsoZ`Mx=kp{@G4K#rzPz`ET zrCJfX&=3BZ2uOfqDP4Ye0g`JbM1;s6bc_47(DoBaEx7?n9TgH9(H^TcY(tU-#>B2@L zJ}cNzPx&V}$!WvvDIi;lz<7G{)cj|bifZR8?2M{>a$`IH>y^j!)AMc3WTAv=tnQQ0 zwoizEp2n@cE&m&T)`2as0S|VyaPzb)!q4*Rtf=F&2D1g13nGzw__-#fhGH3ab&Vop1*WPc2 zWYsMV05D80vaHQC%VcrbB}XRbyXtYVy!kBT+x@MNk>mS5W+5l0yVh$4zUTGt@qh|$ zHBY8bD$X2Dboa}hY0I7)Jd?x76Dxi)%SVT;C$_0tdtW#)oU5A#s<1il z`5zPzb7d@kUk$TFp(w1W6jA}UF}|?UKd=`NB2xj|0<>FG(~~AKmm%YmqEesM>HeR#ip+JINwJM- z#-(fy-QsnS!17y3hjh|Jzh~d3Uk4$DLVPzOClAYLGSYI`C&EopkAvRA}dO5ODN@W?Q@ zA3F1>RpzpGc^yW`kBg*OWoA@K7N;1yn7Ldq>rnv11#r&^;L*ofF%7rEp~w~YN%0$Y zH-HJ=CdR$h3u#21wj8T$9kQnZ2Edz(n)xgM2zm!pL9o)U;t;_28SS*8q+?@ClH8Ve zHnUWNAH7!i=8qG`E!ydriOf;LR_on3=|jyx2`*fe(&<9M)EwH#^HQ64$oU*A&rScgAVxJ z?L=)=65>0Rm_@Zr5|E4%`EeSdWI^V&)J60&jJJ=(i%%gq6Dw%)gKN@-1f)?VssJwp zQJmdJtfg)#T6Y%%mdLh9t}i}(8U#y`v@TPZqKsoLD<+a@uXnW{kJNq#;_A3L-zpZN z`pWWhH933kA6|b2Nmo5_Xx^<9+P*%4$_~ystGHvW!a5y^peLBE_rsb zWrc0~;}*Bqyq>!ebUo!z9FVgtsu@GG5ULPMzM>Kvd7yZo+U)Q=MT$@ZI0>=H!;%)J zsJTnzwi7CSiu_oS%+ir_QS!vr-I4$OlbMNl$0y(h54a~erXowgy;-<{`KNrjC$7|R zWISM`n@1lrR%to6-T{SVuP$2qSyqGRCNz|r4OVel^5l|cl&PN4J6vdTc>19}=^SU% zQXGZ-Ws__J4USX701Dn4H0;ABhJ8tq zQl=m2qCclo=X@pFkb794G@!giS`(ez8xQs7pn0(-D+4NWoS18N3JgJ!`Yo{MkTn@I zBX9lysF>A8v;mrv9?Khgmib>cV^=Xji`v??Up@6recXJ!^_sDM~00%=LzTrez*c zW?&$7aC-QVafr|m>JH3eiMIHn`427|yuZ&UB-0ep!=WiL=kTekW6j;!Nhrqg=#TxW zg1YshT^g~?;p3}Y6-zLN=+TXolY;Hd{N9ulPOkNvjau_BeezJ{CJ-w?2u*{H{Zbc| zYFuWoG4~T=YPRfW!%dnElOzxv+pmUTh^u;FHe0i58>vQ&GXylKX&|fWBHvF5mlX2@ zYiqU^lvYu)@0t9ZipqwAJc^Bl2HislD$Qcdk4QpQ6rAD-58nVVd#}00bM1G+WDu+> zQk`VVEGmpMyXg|J(Gsxg&DOq`ZEk>Qb0%(|8Y-V&we z0Nr+@il^%>rMBcZlfInodghrG1sK7(_1=zWK?e=m%U_kz3$6GE^&-a){kH)}rlfr7!E;@6jEWQmJLm1^pB1YtPx_dpgC&L> zDoyj(`g(a;+6^^h#j85)r;A>CT?RbYm;;}ufs;XDzN91~8#LXxeZvv@6{QYtqh6^@ zs>+CE>A+Uon!?=AcsacDtSe+&w5+zRK+d-=v1v4r{g0X0EQsaNeOB}Jg4V5_R>J2u zq=2)04?y17k`MWP_CD>5p0wZY5;gajErC-q=V+^Sp|GZQCMhfk2#;VQkLAc$%i$Ve2vxPTU`uBG{I32XYCKC^a^GpKD1~uPZ0(tJTS+ zh#^~564j1F;}4#}n=-v@BjHF3%=Mey^&K4u)+L28Yv7u5EFO-!@A89y4OhDzfGpxk zeEh$qA1q0Xo2^_g2oe?YX{Xc=6P*L*K)BYnP9WKX+RdnZS_!S>#*rNBS`W>MLoFlz z#IUtc{M4XEKPPKwZzG{(`GiD`PQW+v08^48=@Nf-56QD6yX=T4P?hJh+sDy+hfIzS z9jh-Ss{?-~QTr7O`5`=zV(gAIP3$_aY=ke3?lXmyY1IKtd2Y%*IdN3}t8!ZohT#uX zA%up*5CbPMVUuR>jUfb;X!%q}4DWBNb#BuZtD7AJ&$|+9Euol;7Mh)I-V!r&j_Txi z%JIs5=DvldLLhu>GH3b)EAn(Z?ClZ)HfU_vj0zMM0f$f#RO7k~zZKd`6;VbF z4k}?uB%$&Z-v%Vob2CGU7SF_#sm&*zg!~x^%R3b}qc2qr{3^tWlr9GRJ9&MCZ#G=p zA*05tNkD{9D-31SCNsj57I7 zb`>cnu<6yP!b-7vWULN0g8A)d6}a{vbxYmnO3FZ?4rY@KldQe*;!%txNYnZxy4Gd5>AWf9lf$H>oLle0 zCOTLm3y`Q?H4K5$yl9DRHexqvgKo6(*ohP#4v}tTWZq0PAgW6YwFqk4BgqMDgR@^` zi%&Iuun_CKuiAa3vhu%qA-NVh*P~He!#F%1iK4BQ|tbZz7oJV7Y zDmacQZT;@a=Pq7xXxUv6|2_M9^w@!GY(qm`6R?vlcd0vR9_aj3ZBmTSF zd0LDp_hUbraTE!%F&+K4ix3A&`JRnk74;FAP6J;?3J?RU3P=N?g)AxL8@Fnh^8aoX zAphksxwxS{?8B-Ee~f8TR3c}eq~3O6YU0>X0qyZiTJ+@Vw)#@asnL!Vt=Ub9NI6Ma zL93yj-nlA5R}mapAU46^ZvmbtD|5#WO+GeiF5?GY|J6{^+^*%^z)av;$C6ahg<@Jt zzQYTqZzBb8>b(}K_0E+h24E@r*HVuRFm!IeWJDDR4;(5lxeLUX{dj>g9L;o)`Eb7r zv6CTiF4EXS9tro1^3iHl^GF^v`S{n5|5&n0(*)DyiG0ys8lL|PhPD$s#esN9-qK}i z%PJl?2fLQKo?rr}+sd+08@;PPy<`fux6YbYkcC&WHVJa96gs}o; z3SWGDjBf544YFa9Z5{1ra>&W>%t~@$VI^AuHM5a`x&fwyXgde$r|!(ITetUCWDIjA z-uK597i+IG52ph$m6JA;J>1w|qSHRnAoh+AL)K4@8?L-H7A*W2K~r;`G&k5b zrC|m`Q@;V1LTdA~H#?g#KA&ts{p6c3%Rp;3d&X_krd}yO7f&=+xnUKcuBvqr_~5`PVw- z=(R1up9*^Qco|jvA4a4>`nhrd4(53ZSi6SG2Tf6|U=l$R;1Zj}A01=sXtwG-(@$yE ztALJHkW#+6vFQ)YO}(_~czN$#7b(@Eq1*pP+7#_{V^f>Op+ z(X1hutcIqFv&`gSFDdsM$;TbH+w#=$Q!U%?5etCfivTtJnPI0A~Z8OaNIo z0)PG~xRV;II;rR$5?nFtfG^g8fs3CqK`hZ*aJo&mmd?Ij+=64CxptOJ*+s$)J@571 z{IQ@{VNCp9Y~+kYl4dw+05-o0@g@V?4s-9|)qPZ%|Uk=?a3O6>FGaXs2?t$cSCxVYlLHM)ACJeu6&MX`f53Cq~9<6L!R^6Dtv~#vO(wycYRKVxo5Lp;{%qG^PpJaugO-dSq zxiX|V;HzIl1`;gL1~oY!32|y*u+$Q)Z>kP=@Z`_%Nb5~9OVIQ?y> z{HY%hlO*RRmp6A=W)o`xC;x>N#H(K1p7LaCM?lyhd@v_$ysI zCN=k!sDk&I!lpih-Vx}l;OYqecLw6{`9g)BC69=OZB9Q}!?!9Vs z+@UC>CR0G*4*wX1oU^m}CeHs8e~cy6a5wJ}=@TyVm&Y`Zn(`|(1F{FZnw{^N%Uj>s z&c5O(F%u9&k?Z7^_=*J=<5S8BuN)c**%m7gV@$JV@LF|_c2T8yaSZ?ZRSW)VkjunL zsBO)Ife&S2{)}>V#ARCYd-?J4k|Mu(G(7k)AEvW9_q~vYf;xa9uJd!sDnSQ!gXFhQh9i&YDW7>r9U0oheMr5l_8yJ$wB1KLS3XUOTs7S60 z)}72@3^W%SQN8NoppFxXrhp3LN2ThH#rU|MT2YZ$mBwuPhTQG_<|(>KtO&4I61j}e zp~|UqM`4dnXVBLCy!7Ff|Kbn6ft~u@$zUe{2eDd0B-2B zEipp0W*%U^-couzpHj-wiUR^M+F83eKab8k7(}DP*_OaZl=)m8eqwUfdG?rkLKBlX zD(|qcfJ=_DjEj?mFSrgIDCN=s{4XW06EX0!iX^s7t;@X)HjL8S<+-O{kn#%H1N@kv z`XpyFDO>T1tFul1@AzZYg4FOEfvl&W-yoJm6SR{p7@@}1vY}!Ft-q|Wb$@J%SLvik zxMgs4xO&DI%%3}KqGiPi-O)BC*X`ZH1ov%SeWj5Y>MBg@ZXcV3rA4(+x2ZpT+Dur0 zEMzpX5TTd}{D9aQu7X^wp#B0YAv{YUuzJX*(zCgj5%|8;N<`ORC_bMa>23IQM{|e6 z)~TUGsYjG4pSUCY91ZFC2|R(Al8+w$xqxww&KsH=>roYdG-Q4+dj4g=F7!lP|zhFLClZg|FKzQzL)3V$ZxM zC~=dK^QWG&#ke7uh7ETZM?zc}Evt8!hekrhyo%pk89WP{Yynmx7z_mCj4lXzv>^#p zI4`;kqTU~a>C;o%gjFE7ZkLmKx~y~H21|7i!8nS+_!WSTu;`{l%^9#%Tw$|j#|iO4 z-YnQY89*Ky%$Nqso8KF{6Yd@&XR^UvmgB0N-v_Z=D>>~xhdy1XK6Pl&gpAF`$+iIi`T1{Zq z8i;~WY8|!R3V?Nfu)rfXp@`q?Z1}1WIAem>mTw|ISqMBK!5o|Xo?_tU8LLwiG%f3! zuN2GAlWopCK9fuJOJ$z-{zE}?i+zd3+_&?D3zNhA|2y;HeE-jU{4*c_%*Q|T@y~qx zGavuV$3OG&&wTtdAOFn9KlAa=eEc&X|IEif^YPDo{4*c_%*Q|T@y~qxGavsS&Bqpk zd?&Kz1;vUgrC3JohNM#ZFS@v}-k_!;C8yGJQ4L@ScQ<6MPaP1y_JHH#{Wn=_Q9$Xm z%tPBt898a%giyPCHq(BuIcLSBW*#I?5_2K(u>UQ}|MfeI+iDEu{Mq%1ZAtfhHi^Dg z!zx3`2H7`m>Jcl3uflKs;Wd$!hMrzXotyTkGH+0z>$pkzmNnt@_2IrpGACV8!G43d zh{qPZ@$w!svif-o+M>SdxnD5E4Qsw0-a6vC4c`g@*D3< zw@|l)3dyODkp!L|JoLoNos);wVU`zq{a-FB2BLfWlrw3(hk~P(7S&5?Z~e+A0p_7* zik2c>?RDSw>A$JN(p?YfyzR{=#c!bvJl!s?v}BQumykw&{e!v?0uQ5*0xhH z?gzQGH_g0{xsdtdedZ+6C?qeRXBLE_aeJ_L;ZurJq#K6m?WwXw5sV*!K3?H)T9@5u z$HAt!Mugwgbd9Mm7YpRNxH#PJ(BotjC~;KmT0VHuR)0T%KAlo5pXRzV>s>%fw!#Rd z`X+>^>aq~ZKwSXkRTWMx1LCLak<&-rsQ4bA`CJZ*=#A9SW@YP9t&#kWp!^@(c&nl` zT>3w6F&88$T%@jE$=E4f&shkXX&U29!GZL`2v1}mrxJ1e*l^A0YN6%~AWVkFLC?&F zT0jkP>VvYSa(G&95oU^Kcv>=AkXFGVT#IT6BIr8^&ZX{q*994Q@@rJ~E&B?%HT5`& zR0x5MOwr^t%UCj}+^?#-!bK>i_9m^TiCpG#8@_Y8e6FH5hMxxaYW#~BEX3?pl4!(o zOx711+{>&;s!%rORK@9M8a)V*!^-(12+JYm<&EsS-+lCKd_mL(gNWGdq(?hLDdSoV zcAjk_kpEa3)$5sM!Je`Gwi*mf{*pq@_k>22;yh_t6&x!ZwN6`nISSV>=U&K?Q!;I;6X8$V$%tGp2 zJncmdr5z@}9td^Ev=M~PE@xZlp^A{-sKHo!b#Rz8&8>NxITn7r@sqKr23n82N=Ztg z8&rXYTXzmD{c`GHfod~(*eegPI@Ot)LKDlnM zTu693X!EZnxHWdGi$Mv~A;w4VB26o)<{qhs*bb)r3OQ+Iv;Cjz(V^#>SnFQR#h?lt zFocKTjJr!+&6ZNz7N|U})f{7t2l_!7@u5rKq9SPAgVVY&n#^9e88n(N9D|Qb06SCu zPcy#h3R)4i3W{6I8mepbJ zD%iDt(ZVT}4StHx^AB{LHld`XMWU@Hzj9p^ih1RX$WT^d1F3_C!Ia%$WJtvv>6SvQ znnl#t!ymGQ{b4bUH&iq%ni%`xWXtmrP=u@x>|xI2iHA8V2hb?FiLg@K6CCk zJvRT#uiPlQI)eYVRJtbQIc~wfT8&-jn!vq|gj)neR?eH#zNrd~v0ll=VEufh@;SCI zS1ZEH0l(P-tjn|mn{Iwo$=Q*BrBU*aho6f_EMD~XB0Z7PvIezO+i{dL=c=8@lx9cW zcT!H02AWGM#vr*b;qb_6lrK+1C|I~;Q}8*6$tvuI8J3iEi=cA9mYuMPF8NRuHIh~L z_}O%B5DiLGm8zPro3L0+tUkW@F(fFpnv%Wq1O!z|C%IXhMQ(#1(xq=cS@?ZTdJ+dC z8FCq}{qKt>(|piPYBVw(r@#yD{cBr!RVp<$ofS?anJOvlXe9VnO@@$aN$#t!o&t?p zroMOcj<}TGY>OQwwB)t9D$fVJfe^~hmyeS5d=Ac})7$%-%st3UmS*ezHTft{Wk?5@z zZdDdYMR?m-EhtTqJxl4pzm?98z7c=Yc(_(Aros0YKQ)XLLPo6{1%;xqs~3XnuSyB= zPpU@jq4uZG$ATNikaR4HY4KN^T9WQRcBu~`#o^D!16#tqTptv!xB+tbny?>~QW1y? zlh$f&8nF%iX(PVOv_BU&S#Z0v1Jsd!W?cR>{m2H~D^-~Uua(U@em*)os`oL3S7)0F zh(K)|gBmO6_gnzJQ=^vRnwhN9{P?&lgZcXlhUOT-`fA7ONW;5JeUtb#T1WTfrPI zW8~gNdver6uI#(3?SvlTv-s(*N%yX+boUCJ1`dwO-K>!}ugcy?%q4kMuh{K(b?71` zUCLo6UOuD6tSeHH^)^@$z&TpwJ7nLi@_TD#IlRZwW%Y-|PeKwnK&yloG+SrD(gwQZ z@~9edBF@{v4aezmro-kJ`nCdEJ&`2kTR+X zx*A%4xMt*bzkI?u2k_ry>Wex-en*hKMSEY-G}=s9Yc@>~$#vLDFi>^R`PagvhDliJ zT-i-vv;hm05Ohgde$Byd;7qZz49?}dzjm^k&mVWB)bA-E40wb1aHwC>siw%S%sBt? zC&cl&^p!tyH8AbnwiWIud@u?vS6iXN3vb% z22$&J?HGtHD{exgUY0|ueF|<~*`u@2!7**h``QgL2&$BzfE!tws3T0FBm-xO0)$WF z^W_9%&Gl2W7Q%mxH|bo*FhD^@YJkFpz3-3Tj(e2B*0;&J2XoQmas9bq+?rLt85^@% zCTU4Fj-CjW{9z0qc>JWyFSw8Pn_-iwgDbp`9&+*-hX;v25$A@mIO3`rvMvx}1rOVx z$9RF4_U+6dpNpJq51oKvc$QIF%CHjJ>Hf3H%Mshc`}yqb`QMJ1GHcy)WkuQ}p=Ya9 zpuo_=s+OK#(^o7ZhW*6Da0E4&dcPA0ImUfCh^NGm2qKRKo-HWG&0E=PpSHURT`bRZ z3Joe{g&nRI&2wk)p$bLE)XUU-O*R%O7viP`7}(kkCJMY}d` zdSnnE0rTYVob-ZhLKiV$Pfv9y*Hm)`>|~;x-th3$a+Q?9l_`b2VBK*w16LBWai6#^ zZx#ySCEt1r;a6u0;aPLUaPbnjX@7nMH|-g4ObPsI@_k)4I3>Z6BRi+2IhG^)V5p@* z7Tj;NfDK%_@v~VcIv7Ux*oC|ceRO;DNGJ9OO&DOQXeV~opOz}djvb~IJ$QtD_^kFs zAAO#2gdJyJ6Gk8)Cld}>AS*FaY;4tsm}uZtQXDKTF|zI;<3C-ll0*U#Ha3pq(;5K~ zsPb$u`sG9|%w{~JH;x35Vruy0Dv1%WJh=*Q>FrmQI|LA+JgE6^zvx94_Il%pYns;V zDBNI}u%s$vs(F5Uakg`VDh(RE-W+>fNqBi9b$ihGF0%QoGwjjMdrLiLFi{QW>?UnbQF~le&%LB>Dax?2BDlSsRk6L?i=t zv>q#|v5bM$eF;4Av^{?9IHf_N^LyNS(p9&4E@0j|M~0Q2C%FWp={bgWG%vypeV?R| z4@`9ATmDAVTFu|Q)-Kh_@zwh4GWQN%i|P0LPhd3h;8Z*@ON|2|iUgwk{tcC`Z=z@Hd#{XRR_JmT2*& z%+3@CwPnyelwU)$KnbOo{;-~F4l|3JL!b`sK9IZdbi!dSnBCL;v;#I9cMh(mix2dC zT4Yf_Xo+{QTLj;)nF8d}Tga^#`sH#I2QS-AeNt#Fz3-H~MwNELk@1iGE=^u5%&voY zOgS^-h_gLJg3D=k5VKIq4tCui08&%1)CHWfQ`n1qpn0NReb^W`2?`>+F2e8U!?hU_ z5+5Jt?**PO+jfpmiTnGkM+fuI&N}bus+^)v54X3Jg>UAsb2n4M$%iUcpP`1_QS+V+ zKvg}W1)u^gDCxPu@5g06gtls#%rGWZpSff!Ouy6w*km<smR(PcfM#n>zCUCV%pqLpa`y?GC{mV7sm!Nfqssrom0$0|6PN zuDqeWLS>~VI$_d8q%AGQX!H0S5QnO&h$4NJ*{0{v68ACo$**=`Kw;{;;qF{|xrc4c zSkP<@;s4&>L7UmVKy~5Waxhp1-*wRKzh-idGIsu=oUT|&?&I%azjpe;uBj|9oU6v3gEM9s%Adl3A?|JNh62!(0?j+ z5BkbO6+t*Z-e}VUjPPu7oL_bqoPrZ2H^9L83fIc z^wH??x|TUOY#tkyCdjs%tr#NtS|(aNN<4)dQtH%5K@WNM?d@pdY>D8q1_Wf_Z!m` zQ=JT%YAPkioL?5rm;~;*U}s<%Y6`ayT)& z=wPKEgn5v<(oUylBfak*h$&C~c-@GsRSW%&$Ka6>X>{@_!2U2HOg6DrW1`Dj9XU;$ zVioaj*766e%wHwXA)5OWViX*8>OU{}YxXA1J@se9w7TwF;shQuD+3C%BNRxEm4mr{ z#~VeC=Y`Wf(YsH#G(~a?m3JiO3az0yH1QWLgJCU76b_l5I7Yd8&1Q0eG0v>xcFI#^ zD^*^CPz`~S;eP5?^9*~dc*qGOnxkNds-JX5%IjwsV^NsQw68W1uxE67)xGRb^kJqv zmA{li0=4<*o~nmw#;FkZ7vV?g(yzqU%Cm&`eC&U1vwh4vUgUTi2y%I*V+kuLvzM?C z1JQ?HI6`mghXYcE{1#0xQ&0>v&Q~qq@|TmDGk)JZEz*wdq36FT6@xZYSNZmpmjbB4 zF{n6BGF%HMR8&wHdb<(VR;sjBiE+f8Y%VWcXVZZd^N`|#86W|4k~vX=SJ~EX@mZ=g zX6U&VshRH<@-l%fw3CIfQ@2DQL11}RPrdt4_8|LY(6Gl8X_uSCu(_S=3D`_j(! zYh>5xHdDLNda&xT#pHa;_wk6fX6*ZBPdfpBL+V1zE%hG6w{~VE)n<~!+i4s_JrX{C z+r`o1Oskf_m?Sshz(8zThbLS*DHTp6Y&u|oF=!bX4K@t*cy6elKV0$BQZzHUl?Ate zjJaLUl-DrQnV*2EvrEs*S^YN~??Qw*qc9es!3w7`1tDF!p2wjaTV#5&9NqznAU(er zVO%&tm@KMTXTK;Mv-~e!93@BF%>3;ZLD;zTnW#k0;JGktb5{Z+x3{dg60QNI#ZKaN z#%5^bUg`irOIg;znNA>mQPunLKv6tmjdw$f9LuY3Bg?D8Is)z4_&0^@5sY@xkLKr5 zH5b7ZXD#uj23@F)@}9bIH_2iU1EF6f^B}YZR&O2fHeHUC=6!h}YMu`Vw@lu)TVdWy zim2avYf&~#k3?vHmqJ=X zN|fo-y>%feGVLXMkm+QtAJZ1I*@Pk-((~br<8^s)4+7js+6~kzrP?#bGDdUp2(=g% z?t-`J<3l|&>caE7TMz;7>thxRl3vDBc$QL3|7v9!lxpFo3&Xy-Y90^kPy~>l!|bBC zf$M7aBm*B+ng)gw_xXKR6#FVCUF4aW6tkoKUSCdpp(KA0Z-x-yKbef&>Y?QnA)aGJ zcB$OfHrJ5f@!e~ilq0qfVZQ1`*Dh;T=FCH37(-5yUiN$V)KESzFO{WOnz@W`rmML% zsuu;%9wXpN`>9V}FDiq2*7ehPl(5H@d@-mMUY)QAXY5nXJ1RymfmSgKN4YtH&1Z;R zccP1Cz^_>g&v%p_lnx`)Oz4llzt3)sCh&?q~CLosU)Ht4R&;aW&75Lj#2Ar?7aw zbFIo0oIIyXv`Ox(Eb$l4LI6{P-ANQ8zXY-&rtK{0HWIQB^Xd_ zS$cL^3W{)$BOJId?xIgL3|KB^F791mtO8-R1YGDw+Uu*7)}1yNAX=NK_MngH+V@z^ znTCHOt zz~d}^iR82hE>_K~W)gIS$2qyP>ZI|$Uey=~?i zkCGSmNb~z6pJxE4SQ#&OOwsU_mE!ACvk}oc+c5w5Wwrd5KWUp0KG3u2itD>TTxDK#EI=zDmE#sIi5a=9#6F>?3Pc#pMa1Z{}j z89pQ*hSyd==z07mFXCf>0q(gC0|rhiR#pZof1~JTqd2D3^=N5n4eF6-)ZV?2ILli$EzAnhjVh%Z3~90OiPrNAE!~cemf#cs9GLuqK?bJ^WSOIlMkznbL_?vl z`P}rfRmZXuQMOHtAm3;+KFL{6{lxE~E=F)GObr4D{`v#^YWeG1Pq%v<4X~WS?_=Pg zuHi9)m`B?61mxo;<0`R<`xU~+gu9>2V#~X@(rHDv1R*$#QwksE@5iv|0?3CASHkzf zJ(0uD_WAHT)%SbrAlvsFTsy<}`!>z@y9*j5dF+ z`zF}0W+3#&svsZYyqLU)c8wEYf%ka~*sTlA?;wX3TT)>7*-ohzCzv^~Tj)yn7R;EF zYIf_dl4+cU#ouiVcwt0-ZWOE!r`J-sx%Uj;qtS`pA#)sIoa1_%c7;Sa)iMOGUf05T zax3fyUrX|cdc$_m65cLQ9WM}dX(XjY{PIK{eAZ4 zxWhXd*BBs_Y6h6E&t7q`z50BP12IUKGb8mkD7uL*SrEov+-WsQhNLxFBboB=XU$(( zZHogS`E27{_Fhu5GgQrc>n-6dIqmUqONR7XTVrb1;?HfT9W~{Nw`}E)EYbNR4=jNAm(T2#poa z_q{vSc!_T6-BbjC<;A(}>02KRNGm!R?jvRP{X8mc1k~wbTq<`zV#tI$tRBb~4UY*= zJ@l(J_>{xTeQe5_Lr+jVQmNj^oDG85O)47DT!~l>b$bN8ni=obkK4nj7vbH7-(N9%A}IUI-{Ljkv6gb{WO$QT?HB9gy$ zzJs0t#g5Uvoy@vV?9W`ZT_ zMigvy?thxA`?SGlsh{1;TfywVY5Fmb_i3}mWrG&3+zMeUGZ7P#Ko&2rS@wZqx=lG^ zx=ygpKM&YDM##$l{(}|woyc$5|4I^Rli287E^>23v%0-|%WQX-4N?f^sAKShm&Ef$ zo`EON5lzTGMnDI2*`w%D)m7=zDSHAzCT`I#Ow&rWnzd7yP9w?FFZpCF9Ix?X=Nh6r zRzG_Rn+}UV`XE(VDyN{9W#^Yj;q=7h@Y0?G&}}kjw&Dk0)npqt>esuktF>r40R<$$<>Hf_m$R$mG?xjQ!%v# z-TAC_=wV(isZ|wP@I<`N)r3NK#iX)X!#o(~-6ncb33m#smHz_A1DF-H&B))#ei~Jb zcTR6B0gD9uVt&Z7gQ}*$CPG%etK>&eAsXx!il}{IaVV|ZD?BD)<@1wrtvJCnH)nHgz=(gn?6y<3cz6z04+bgl^-!kYCiGEEKmP zn^~-b&deG;O0blWP!F>o+}BFCk<_Gn;!mz@H>mVolvA(s4~`im(&Wu;^ivAUidotX zCDURW1gPm?EC%j;LJyz5ZmaBfR<#89Ywre`j6y0-&T%n+{ME#!**9dqjAJizi^z1+ zyphP}V|IING#V2uiap>dy3>TMybOarh^zGoTsk1tXXe<53vsdsb`#gDQyXIkA;*3mzmcK zW2BvN1vY~QZkm%*5P_o^dpejoYJ^U7ukeb4!&S=+HzUa-UT$=Yf{Tkei5vgUW`%=v z!!gi}%l{(loPtD&x-4C`ZQHhOyKdRGZQHhO`<89nRkv(+{oT_KGZPt^5qTmXGfwW@ z>sxDY^D2)BS?Ji7ZqBuB9Ss#%_v4%waMyK#C;ICj_NW1foQw*o@CPQa$;x`OnX)ul zpHTo5dz<`wrrA~FcAEL$M(GuGR87I2IBzLo&wSI@dljJPx^_EQ-hek?$0^IYxi_Xp zqYc?DnqpsNjNJk6-)4u!3)b;v*j>}?6620VKT(KGupAy>NDTx1Ra=aADeKN`71^loT<{bgr1fCw~F%)(j>HaIK3IyX32k%x<* zg_^EHFO>MrqZT9f00e%E)m3xw&wO$FRFlChliC56vCIpckE+&ugtXvO4;{cB)o$Vf zP)83LLN`{*30K<%!SWs^Y5BdIV3}qbUnOU2-Tdg@DCPz;1H3ZZc&gALHD8C$&&S7i`#fHrmWQ+aZWr@@beG;> zzb{Xd`kL5(!mW=UAL;1=x|_DT%TEfk4`Fl@E1mc5Tznes66*Ho)2gFlO@t|qH#-~K zldPy4Wwt7y*%qocpEQn|^R3wIxU$+n?$7IgSnVNUsB>mE>O5BXFak>5HdsbhcPyNg zq_u^VoRJs$vQ|1sh}5O>=;XE%Ez)L2s1F-}kgIx;^*;u=4*uyzw9*YoQ5a3^t)wI=NACqF`*YWf|FAvp)8{U3vCo{!@qT*t#`pXEv?_1+ z)7xR<$MM%LADbOtPVK7Lh>Fi}bfXR&v^tKq(*~~B$eM;qT~ezo5ygTXTA&-D<})|A zS2?5bv=Ty$SRzw(`THmD#TzsvjM(a?L2EYzp%1_y7eVop`Jhj{(V(;#-EHAzU;d6d z%amWu+*r-}jsjA@_e1lAAmbO@nlcv3k>BD9eUG|VBe(@PH>Sl@ec)bmCZS})u}N4K zYsY;VO*yY?YcWVSaK>XCy{inm=ab)E4p)xYviVUkgCW@gV!pH;m<>y5W)929h_%+_ zph!^wvaEb&CO9LZXr_5{6(BbKq8|$eFkWc`uF}=;&f5H1%BV7GzVRhL2WC@f?#a9Y zqv(K5{@|Zw+yhQ>`|ryt4VhZ^h35eVJZ$?@lu@m0rWiAE2`>s$ZF=sdp1?Ao!U9An zjH*Ynca7<(!PT2jU>Y{q$u;$`RJ3ZqC4i(dMlg|C8gOS-pt+D+E$VNk-Sq!aTFFkT}&x)h176!qOurO z8$2=BQLQ#3ku@>KPOZ;)J-A+QMAq_Ty3r3S2$lgPPu)G53;R!lk8bdPl-6^#oX?-K zF=m{?9}?{e>AeQ2Y)8ZMO+9mq-CwP>YN^(5`7_m2gH&UiH8eB1NU>q#FqPS{4KeZj z*3Slrr_-RTNb@wb-DX_;bXbwP$A!*hsH}g&>wm>o(t%l5c-77kP;u%6S|z1RNn@^q z-U_hVjXA6~%VdTYoSDzmCL)@~bG(sBs}S?voTDa5J}25ylDq-takCk-#Mv>DycpbF zl04b_^`Z$gqeAh{<XhA{S7tV3X(+iaxsE_fTUzgnYFDy}!=q zPsKVGvObJeC9#Sz>1RUID77M0021t81TXb~6$QI;eW$2LdF$OkkBCza8O0=U? z7-3{qCvLA7wM%i&P%t{>Ae#0&&WyU16G5HvwKLt9tI?4{O>rSdt(GU`eQYfBnRQr` zA`Cd2rh1`%&MblMoJv-L=?;AP@FDd0Cj2O~h4D)GpTER~##ZT_(IG1_aMe-30Tm+!HXNRD;_Gt;VEICPYNjV8%ckt~$U}1AXqSW@87Ef-y!H1?WuH?6 zW|2Q`^rI+0@UtjCH{PYMeoVtZak#rEKm5Z#{92!VwEdzJHsbO-8id*MT=yILon%n! zz)3g=*J7tua4cOf4Jmby^<7|n;*_X71~&}Z%QAil9_ajQkb-<*A^TA!8A=mD*zhhCJ5^Eqe@n#z_dV(Z6Z?` zx|o1v#*Wlb<7qsHszyK+B@0d=iPg3q7a~n1ZpL58ySADax$K5>HV_ z!j~RHFhN?4usz3Ppn1ho>5}Hd3*_~ZWXWB|9qphLiXu_QFTzQ)=lr4;hMm(iNm^<| zcKSmKtl&`Ha|pAf(jvJ{M7SzzEq&l|($d}BFBh4=Kvgu{v6u|iXtARVRoK_>`H-Ok z$5er1G|FCqlQeNRMm6f8*l_PU7P7GP{Bf}U&C{6~Zt<6{j961i{WFzyox3LyRF5Y~ zm=%hTokkAlOO%o-A-uq<4PLKtv+c>k4XDT`ukK0bjvwymx#WWfo<&dI5ixu8-8Sn-SXAjdvg4NyR%O zS+_jk7CM6eKore8XT@w=_A@nLGV(U-tn9VHuv12z%d8ltl(6|unn+qGVUiTAgN{0z zxoIJwPN?XmP*4XhGCLdV9&kq>(meNfZjmlak0>ma#VmE!re0TdHZg$4vUHMd999$* zUujQpV0E5($IFvYse{1IypfU5lU1s`sEEI>+I(2e#dVxY`j8xj$~~eQ@A0BCqxcAG z)$H5s%kOFPr(KNB_M}e3#cWJoGi5Er;|YuZiiK&eL|3A#FsiOLIEe_)y@235QoT#P zuzbyj!Km+pZ9SYO#rQYAWyDr+dbAH&OO;d)2mHf;I!WF3jTH9ITR?%Z23~ z^tsQ(WTpY>IPvo1^(yQT#h`k!%fo{`Z}t>-ez@dTgBq@gQbg1-A+tQ$H6&$h*Oewh zqXXp4uiW+u>poRg0TAlDYj+^9 zDPdbs#`Y6@` zK4|4nG$ea2H^lU$ON}BL^VEqYa&}?<^PEKG__Kl`*d7-f8$t-}8Vw+Sa_cSwPHKx4 z+~n#c*n<0#lU-(;z}fSa3I#MuEK?<2ENX4){=_1?D0m^2n|2V=H;5g6uF2~&IlK-S z`cvw50ySKi2T^U^air_$jU7n(A8I- z@P-nPl~xo;r$b}!vZW#=$as&*{`OKa==<1fi0NVmr&H0qKDs4KuW)MmF(s8O-qz|? zhdf2CP+s*cQ+eC1~VSDqbb<^LeU2V3!C;=60Xtg)a6iJJ9XGh+&7`_$sBm zQ2Jo1W?&#wLMRVP9PnPWnh`7d6gTyw{>}j0OuD&?ti;Vsd}GbmG;b3u-R5glh@_hHGL0ETh5|(!0N&zNq37%&rA2l@ zWr;fjZ3Vn9*Qvs717;0Qqc4MypK?rL(57*0J}>6Y=PjgqLD=n1FLzVN)rIkQp)8Z# zd(~1XeL=N7_w@~9xHxCaE{()z>*R*UdEvQ|&#CB7Uv2jKhB)K-bG-p^_3bY{CCgw- zR!TLizav++l>%HuOnZjR(+Re`B30Qi3yO>VgR;Vx5Cu`on2g?Px6^%`?%tPjZ=76Xm1Zz5G_7!Pwy>{_8qM4r?fZdUA6W! zRK@uXH>}E~wz9hpe{~vx|AzX+@a{#V&`65(_D~E@WT3W-8$OA1(T>;uF-kty3Zrr} zFk-;Ql>p6lTpKbEk}z*YY`m`>-N}69Z`#8e&;#y+CRYfch225rqL3+$9*Ze&QmFy0 zEZleiyE)eYmCuS~YV&V*?MfkVE9}-3@Fh6;sgMLs&}?hom~wWWstKScW2d;rm&W#6 z%|_<^BVSZ1x5^zHD$_ZW(yp}tvUM2!u2{nDY~H_ zXG<467p;$aPoSbSO!_y=I4_}gac36ajdt`_lB7^o9iHQuVu)PG7N#By3ou$ zDHgm*yQ!}{@ht2KE6C&hQVnG1@LueZE(G#R*A0L#D@tV``wWi;=06`A!M8 zd7@Lh%7Hz0CsyHIb}hW zT+>~K`eIB@nliWnP0s|H$F^-r;Euo=&-j_fK3HFssD8EXRdHZvQ0QXOp7RVtKh-(o z+x!TN5bX%bgr+y`4^T-|_z45*w~5|wt)|;4lX0!lwFb+iGqEcHa>rB?i6eRuPiupU zcSi()yxlqNm61YLLp`%c32ZqA{2AJ2L<+~`2=3z4IVMuhK6T=z%_eI$1w0WHr_0%1 z9~(X%o%jc}p7aXU7B2}GcjA$6Bl}(>dV1EQaymVawgP$Nm!*w4k{l6Q9mO1p)dnksYxJ~xKqZ=NhaiLj3HJLCb>#}ut=_(BS zT;Vy)bLDS4BQqHcPFun`NE&4zD_>?(`YwFRK-3vX8l$mxQu@Ek`1AoT2yA4@1s>~f z6Trzv<2JxaJQx(@V&>S;$%g^Mm}kcb`PpCL}`s5Cq(Jss*IX!!0ia60O|Td zqvW^XD3}W7u6)Ylde<+HAB4c>iun$w59VD1r#VqF^UmJ?1?+?cAmE2;5tA|IV{OKaz_7kF&;y^o8oUnBv=r&OVkQJ+nK&A?Mzwg8|OC!m? zUJqLo2b|RAJD~N{DlX6qpPDG%7gw(ZYrgKp!)xCWu~nys%A^4CP@HY@D^EWzqENG6 zhM`Blu;C_?QYg;3&ZvjlGBFs}Ouo8&?28&~vr#T7dyriebyg;!wQLNLer%w~@0gXO zS&IQfk`q;RmP?`h|KlDtJPt|F5sd)cwWN2{KM#u3X%Qry3J?_E)_1|`es|CE_>g14!|r$qedle7{a;=Bz*I-*;IqCvBwO9VON^^lM>n8bZ_$iXlFahX3%L} zcWbx&a>YfEH*Y{!D!mjk_gy%7Ks87i=E8$bPx{^2`pt2;V00<1-+kE$XyQYM{R~N% zaTjFXo$cyrNE8g@S~7=P*}{&XjIX_NS362w3+Ha!H6f#{m#?(dTf$JTdx3 z@`X%uywge&APN7?hHo{_2sVz+4{Z^?*J6WTDC1yv7w)7`yPNKItgt14ib1*R|$~hYKfOOC@QpHj%8Xa-qdvYAs#9wdd&(&p&%4%sS zX#3qo^6Mzd9g_Tfm-)JQJ9xuKKcZGll-X(+!J*Y*?D3Z>0I?kd?+45KfpoVHA|8k8 zoRZ-jtkWEj%6&9FHTM}VPZSx;RQtDQv{wDzz29Pm^Vj=EK8$G>P>_(VXhO0)YUzZqDpp)CV@$;|$u{2nODBBu_KP4vY z-mjmDc@W*&;QROv#uc%AE%IP79`Y8Jbk#u^UI9YMBqN_M^l)YkRuS7JxyNM$_NrfTF6xfLn|a*;v*-PZIP@vbdW0t0AYxCx(zccK?))M|+S2TMv$>L^!| zlyUP8tL7L4iW;c~G+S$?xG24aI7p?Iv>Sj*HfpI=2P<0u_6<}nI>)ZrA#~Nuqxcxc z7sztT?*H{(WB_mCvMf2TG%U%C(7GpeDvd{kxwV^8!EgTL61Ga5VWMGMj%QYq_aMKwW+Xy{H8%Cmqm{`&@v znwwS=5G~!*?Ghl~-ZdA1#>k$gO3P4oA{^OlAt_8PCu=HeZ^bfrzK8NOB3S@UP91xm z=W0!AE@WOm7( zx;+tt#N!VZk6hu`Fqc)|{&rI@yuIpUVRz!uNRF5TvlFw3k;cv!!xNk8Ub&%8MJJll zi=sIqr9qJ+h}(3By;%eD>>Ul5*}?}=#DnsWu27U%hmg&ZG5u%tim^};XEE@ze!euJ z$uKPHb%To{)f9c6Y*nk+uy4h|YQ(ZVimWBTAJ%)rm9HL^OMts?!-}_Xy424rxnGJ~ zmOj%sw!OnG|BvimNDAU$iOFs(w`31ui!GfOYK5A+ZOQahF_!N5d|m4yM8O+oa9ucenc$TL{A2m~akJ?X@!yLDeqob5k?xsUvD%fOD5T}$ z)=_Lzkd3IDT+Hr+u-CpV0Rb}%1zi!bQD)Z)rGUo9xHF}!?)gMl5|S`hcAa*R=$w@K zLLwF?uyAZ)j)?Q1Yzq?bt&;tY`&a`ZVgB2hs!$T^nvMrwOGil!y z=kbUZHZLTDXgUKD#iWhh488&vQb(;jkbfrLq9TaIWqZBH<`L?vv+Zz-FaU(Z!PA4& z5l1>73y1yp%|FpdY7WAoW_FT-h>X3je+ueQkR+u>AZAlKMWP~$y-U{ftp79=fpxVF zNIiTS0QaX}F$xeHr$%(8Zz^m@plyRw3x%ghLS#M#U!SR=HmGS^&#z~6Hn`DDF8$z( z*&G|1?|AEkCkldfGaiWsD;_%iH$_zX`q|8H2f;;Y=Te#?7Z|?9aJx+iV-nsdV;yg1fy8(c{z{yP^BcXF2KZdeM{csvY<`g)AI4DzM@S#e$r3L zrlhz!W-9yKPi+~}8H6J(j;IvaDj*GAC~$LqW~)eWjY7XN0`)!c`0gUvb1VB?^dxBC zcVW)!bNNJ4IZrIcBTSJYUQ-uW@@!MpM-I8EmJsSq+FWsC(~fp+FCl(6swi<;nQF{IY=uh=K}3_$)#%w>^< z>t{`gSmb_)F_-Q<(qs2ku)j^trr`ZrW&SPAHygfl?WEvs_jA6G)`rm$)Az@6c?2Zi zsNb6gUpHX8s#dE;ATx{q{nag=HkT-UwA;dUM=YHmRu712ML=GL_WGRev-Wxryyx<3 zGoSNn$WFNEi*R(gN0d5_=rlzTrZC46Nwut!b?02;+LnH! z!t}9#jEDV+ZRub}RtL;R!S3X`#(B?^ba-UF40_DP7GPe;)F7sJEiR)@=fjfG2_63Zv2tA(^~U6h2hoCjzdJ zn7F`)D2v1gf&hO>-a89WA|K)`@0?y@GSq^3E(F-2puF%K8CHkI(x8(;T#lkh(MGHvuLD}3BP-fp z&@_<+r|S8%QL z`vF16?h!6F#3OR$ic4PsaII`i40ul=h(RGib~HegwO(OfN{Yo@#W>1{5$9vJY=Hs~ z&9R&|p^hoiHlP9R-}ei!2?mkc>4Ig6%VT<@O`MIPtO`3LBm;3RORJBX1p&3kZ6FtR z<2{biUrHJmFzDAWg(S5RzBJDm;>@dq6@lWv$DESVzLOJjgJBdd7iS!^qDgkLS^L*% zJ~W6n>`Zfx_U~GTYXUBW$gQdgHafN;(b&wi&$jRwR?f8vk~KsBX4<^~HUS_#$)+Ew zp~;JMaA;k{q8~IlV$WF6+G+DPrB+LA zFxKzDE|nueg?Wj^)L5XX8oW(>E|j6{=to-JV>MDNaE^SeW3lbj9scvX#}$pz(Y|cL zAfFOz4S%}3OlBQ3Prke3fd1?z((ExH23?Lez>dzs=5`?JZUza>i9PhX5#dh$+knTm z?}R&1f1KWzt0<@d`jxxWx5l9iQ;jmG5-1Ga-Rl0*BMAs(d4GoR2w0l)v)re(UbY5M zIy`?7CPEh6Out1ZqZFFqlAedAtZ@DdeV9#n?VCX_oOkBhyx$P|)BXCbs+`ymmUdJF zB!D3L7l6v5i?t|`Su=sVlX=g1DO|z)h*eb_SJrjtd-D-48kL3i8D@?Jd06XyY1tgO zYn~wCF*b1s1BdNvf%XT_AymOD=!-FL@KL7|2*c0?xZ)^fg5x&Vk<*d%ZFZ@wlJ0==2Ak&0vq`z`G%xbQZu%g z1;>f%q@emTTk3n3gh_OzPpW6HVBxaMl>=TTvT$)|8wzd^(mJvuiZ@EIRlqIA$KE;* z$-FJG!Fsu|%_k;^sTX>;&OO4=&16mN*5tHxw!|jEiWD$skrb;c4YcN@1*qClN?#8m zWQJ8QNF*_frqV#PeMQ_l6wBchNJa;ML8$F6Rm#%azyO|MS$H03_BVLP7{SVfogcv? zf>&@=(BE~LEyvGUx6gCIv$Uv{QO(P@ub2gzEr?H^LF2;>1LK^_lvcr5z7T3e`XTAA z!I;i9?xJmxRB1fc{2;)AQlT+!GK?&O z07w1wuAlD( zBKMiPKO`mus;L1-C{YncMuT0DdCm}l$QlBGX08+;j|S?KdQNyW9t-jl73YyJUlrhy zdhP&kZ;cqxHwN?rh@vjs=+<07k9ZER2)Tq`Hu3S0T*5E%=!`L9o>(bYd~M)RDM!4N zGfwgjmvN>2?{R|UWS()gLxLWnWHKUT^86ij*hp=(|HL~AgJvEGkrfS(n)6T|2-YQX zzo8G5MD?A$25xcPxmO%^<#0uhwss)7)l8U@X<&NBas}5&hl|CMpU*|~( zPDn}&$y^FHXO@}bbY=kz#HT5ffcia2Y-i>w}uRQvjl1zY0e68#R9N_GG7kA zVu4UkSttu$yhsj8_(^Z09u8vsE}}1!2N7;T0-;RP6Aq?txu0VAIK^>WLf5R6y!1>B zOg}6w*x)Z6MusaEG#L$^iU~{gpb0#|gCTRG>`&#>>MvDfe`6SsT*~~t$#ob9n#ux2 zF)lzb48f*IAc^*=D3WF1Ps0AFDDp70CtPEfGpZ96Nn(@H7BB2!w1uAPUAceu~z6lR_HJ( z(J#Y}R%^~OGOUqas1aTW_fO$5IOkU6BVKskh7XQq{wxF+ToMIVoCT>VDmXu!1}65^ z=scA0;stfP6gbcvLye+yneeqhP@>=r4IhM;1Ot>xZo|wg4%d6O%LXpx%L5v4?Jz8(kSp=>rwA?=}53BUq4$5 z6c$R$EQ`~IxyIelecNatck6TFCCYhTY$ZrY`dzT`U<)5}GdP9MuC5Q@(aMb&w&O@6 zb{RPEHDS)e(iH^WKt}&&BtDW0i>l`uR9dV%2aXfcfy=(j5BV9s*=r?vWk^v?su$)1 z*rEqQ(JV^;*2G78U|bI+IE^As2#`o(}wcfq@@IWsjz^{fz>&Vm zXD=quz=p28ACG!UsDEpI+W^{(sOz;7=sDYz7+QakC+$27$% zC+tP192&5YJuAz*Z-vCE@QGu$scA zefGEH+nn)BCaGpl=YV?fx($WY778fifV?AdH1lk*gEoVVh2@8|sj(02=KGmVlVDYw zqy2CZrbiuKylSVQ9pZ%finq4cgi-U4_ zS7CyVzz6CvLK!S0hBjX4W$CaEF^c5}jed}VOt3DxfnB7i;8=GRTR)cnDi}|bW7Xdn z6SU}7G&E{+3Tg4mtfx|^yRWMFFK9BW{>A>#YS2$@1dSRA1xnFD3l1!}=Lu`R^zIfT z{!QER6w$5p`u=N_)^88uWcnA|H}W~@qCkaF(EL~^zAtNF zd3+RSECnvo3aQMku~<#s^2rRf=A4fY^$p*0I>xYk>F%+l3C@b_xC+Ns4_a--$_Ma_ zCl7{Z{c#Dg$(HfBhUgU7$$3aGEmVV0q_CMlT4k1SPYTPdp{F`673b0+$VrBIA#~E~ z#cvl?8L>M;hS^bF4Z0_y+`7<>W|OLUn?)JPr%v#_QwD>?IhqPaPlONLu2wZPWFZa5 zh&Lsg98S}Mqd0-8PEc5nLI+H!`Pj&BwQGx5l-Fn zesmB^>+wo;AXpC(tTz@pAklMb=RvWk%!^nMEL=*rOV<4b@qD`4LQ=rDq#~?)9eDJpf5Y&FO#CR3d~% zzg=SPEnL;C7{*F)qwhY!dTh89^~SSZLPq#BjyDyi|1zHAVWupLQl|ck9NklYEZunP z4A@rS-vx}lWTJYAJ`3mS5n?pxqgFg=_LM{k%cz1Q?!`oHs5T8{Gh{6eLND7CF$*!Q za%aCRM3LK0Pee~iuL6-^AvdgNVwjsX{Bk5C)rr$%`O49(kj+^%)Md&P;syO$Eq6H- zg(CPE?sQ99>eaBqZ2D7Fs>?*>(rFPSgy|s z!oa&b)5e2Rl`|ujSHvG9QWu}9`T!dWp>S_v@uJe^( zqx}=H^s(m`?{-z$&=jh4yDA1kypq5C1lMm3OqWJ)0{o(oY{{G07Pcq1e4+}PR^+o; z%?R=!iXur5rKYuSeW?x6qjMRgi1Y5s2uSo$$>QaNDjjr=YA%gyruv(4=E)OyOb55D zc8M57;W%_6mn&_4I2y5xq=t8n>F4zMS*S(kg-yQJoe>7DrUWBs-367z5C!k#Co&4G z_D7GC=_*6EwZCqB!F>NQ7F*`Oi&I$p`*R2Vphgu-6%cARVK0S-^y6NxWjByf(K*EC zY*xtaYOH^}T?UMEHc6S-X*we6!;!|&1)J=c(L^)UlFeKL%B^48kE5xzYS#;ZK+l!) z1gtWthOIT4c?yn`@<#(SBoZPhkaEAUj=_j9tcu6lw-q?Wc7k3$ymvHqZsX_*KGVjg zZhQCMj6_#IqW0tt%1MbwNB1-jIdz&2jr_;WScV@>za|TCE6+CU`KWW`k#0{filFj_ z_Ho-wazh5afa@g!*8jBFv3ba#%)ey1Pyfdss85%eXbT?4=~}KSog@lrUJ%LPt8L6J z>*!&SV$Fsc61H+Wd@#vRmkT&gwYD99Jhm3$uK4wl7JH28il&0C6%N=?h^=OzZgRO& zhYThIjEj2&wLzR%MpF>PMtAs_K*Jbo;@W3I8paU_vtZ=Z$vi_gz6z>$7fdAul1s{n z5tMb&RGvvAGMpq9O{Fk=` z(T?_H$hN2Do|IWP(9%_+H0YAy1h2cSVAc5!QOz1+4o3+%58CPsUSkz8G+#+EUl)to zJy%|L2;pM(t|Gj>FQP2uA2mS9v7stXb0{e)w`SN=&p&6R)+zSuM8B>k*+M7=uOpdM zQ>X+9B+Tan2LnKlF*o7Lw6K7|#Ck#X43>^v067pNMCtAL$k3?Mp-Nu0n&;Iv1@v@o za64gw*HDP|MF||;0Uud8G+%{n{RaVgM0AYwr?tytoJnwzGnurnWIZA7>G*%F@*a}8 zUb;~Sar&rHxk^n_ot9iOCsAfdkxpy^yn+qNw1+s8Xe?={v-jS%m5-`SI&@Hsvm%a7 z2bd3pJmOO;bsxdumJaS>C=A@`XHR?wf_fL8a$`!V1T3rLXY1~os#IWoDz_LEk=n7y^RR~7$ z5!F&+I#eB`92nhF5SefL|AsyuUiA7%>QG)-DDfC%5-;A`!|4i@7C=zvX{iM(8BDC! zEr*e3B8wp@4Mc~fO<|8Ik$l8I8Vea97|^W$;sWqeSPCFhxM|o(%s46%S?Uf|>x&G= zFgfcQB+C%p<5)eY1#U64ibj#p*#m3FuWAmrVgB2DK%f1y8gMF;m{TtOtG?AdIux5l zx+SxoJxh#mNs=>*QsT(+DQ+kHNaot%AX2F94L9Ya>(ywe)Cd60bm?%NquzAdOdt+(duh^3`VWN( zZ;s|AeJ=;hT_mp=&pnZa!>N%+#@?+-7-#>et<&b1)eTDeS|d>?w}6=0CjHphnCl66 z00+!M3&W3U*i}3pBMqQJ?hZOk6~Blfh$!e3=&Uk4o~dR5PGkZE;ko84(uD6p`Z3U%q_f3tVJ@|D)zYK%+X)U_FcHgSG2u;KB7v%JvW}Z$Y)hG|Y>3lp zI}1?e8)+XimY9eQxl@Fkr8fNr%>0}F1Ih2CllG6&Sd{0hExj1&>5tFHyEX&wS^bt) zqiW7Mck_Se%iwy0r6?OBSXkwLA1li4k8rU#w$QtPCU(Itw$!A%G>cZSyoaBcBOB}M z8#$~9Keqc6POgC1OY-w_5lc&*AgE3)>iE7A2ng}v6B%4|9+YDz0wTIY3+)UO-qUFw zA}!Kx*KdosKIiUYuFtyxTF8-%+}`A#h_J*l85Re-3RcAc7J9pYDWy6fyFE!Y+kO1A zAhyu>TcBJNk;1|(&|F_ELID_iEsVmZ3@FKSd+8oP{JU7?^Hw~=L~@^bQ;GW z*wxg#Yp{)a#Xsn*Nk|<&qw17lIQyuTpf&P3~Ls{aY9L^nammO>BQios@QdT$K z%P5F|#c}B^3MC?R8H?>Y$a*1^TuTbN+eBvC1Er6!qRB1&Lja{VY`h$74?7jKqm&!q zU%Kea)A8j7Mp1+dw4@Y}Zx0kS@C{Y8 zJp}AWy~4%Pi?4JQAx|ER>cY46o{a-r_wpykDg}P`ob#>BIe0tdS+aV!Q5&sE(y}(g z^JP(<)nw8$9eP@D!}0SxHgzb9VKn?)7>(i-Z!WG@%(#RBK!UBFAeUK5C>M~-(;R`{ zPPOd16Hl5NPb^|ZKrSw8OgIQC070*@aHiZ11BQMqtWMYGZW!HEkE?_rjhRRXEY(|#jX7~~rL9lN~^bh|iND&8{-@&ZZZYHeF8-PsB`y#`~#|3qIS(p~GW<~yxotZ{6y z%ofo^e^AQF)zLEu(#L)(Ngba@1^toSE_NQ6*cr3$`=#T;H?I0cv|3qXms?@=;l@>B z7W&fN_iLtR6ZEOGfh*_Gk?%Kp$AMhQUAWrUqS4s|)exsx1X$WBy&W<`>#kt|zj_sR zOK2w|k8S<_I~cg8GVm|l-HH@bqY9$vNixapCR8Ea7sRn#L7C{q5(aQO7`$WLKtp(< zJI*9(?B8jM7lX&AB@1_KmXubc`sPc)an0bK2=wBh=cwr!8kRFT$xX7wg~cO{B_-C^ z)vmx39!C9Mv+(Q7QHa(kUOG>7G$?`-mlEeOFisGmx*lik#tDF+eV!6KW3Bc zA2)!GMJd+-Hf;eb)^2QI)rDjsD_95|f-y)~HcXdR$?giy5wkN{E|Lb*4GfSvxJ`8t zV0Zq#JujNEo%t*F2;lMal{?r4CATmmzP7^umu5`u(-e?5TV}Ttk5QuC@On<@!iigm$c#MrE|Mfby z`T?`-abMRY!0Wfi5)qm6Yd06Ijv4*qGtiBI2Y>a@9qWg7Jd{ww&JvW@8pBPzXLWkF z#RusU9vmDb2%i&VhSG;XgpZvwnyn~?q+bcsqyZ_?9x4I=N9~qPB`s3;dXF`9h?~KJ!xGj3s~Z=o%o`Ws%8>BGL_Xg! z@gXTR8y@@erO+F@xHzzR#=|iUSvG)R%zF9K{R|lA*u3Jqn>eiA)Z%Qu~z~8G@ zz*E4%8}r7+=vZjJ5o})e-jsM!YCM(ci)p1k3cwuc)$7uCS^o)q^&7SC*B%2sDsaeB|% zFL=p``dp4A0LqlhL)>z()@9oCcN#J>IrsL{*jTu+ym1S-kGgB7tvzy{D-ov1}H%!vSI{OxZqKCOZ@_cyoL> z3h!_MFYl5d)qQn0(mIV~*ueNC)M*j{uVqo;4mtFr7LzA`^}$6LQPxS%xBRXe=#u^d zn>tq~zX(- z*zXCp$5croj~QkyF#S4eK{^LDW6j@^sI40%Bs7+WGaND%`Cng-x8L*r=!@CS&Z|D2 zi=-a2Q~5~)J8)H%>!FYl?wSg-Zws}+<&-x8NOi4cCg_U)Hn}G4+m-N*A4X^6e&h^0 zA8LCEJb|`xX}l!>|N2S@od{Ljl^yD_Wx~S?mZKE2msn(tgyL4OB^LO0X{C9i;JmaO zG{h_1al(D!LP`!wj~xq#fb8e*XLlL_3$^I#XNar#w$~~Rd*EgEr0fTSi?12sb(2s1!r%GyLp;Id1zaw>CJG-MkbdL zvm@Y9;TP=I_{+gGuk^))$8g~4q|w<0%__U~>QYb88=zbuWV6L=RV~C(t~%+XC)dhV z=oJHI;X)`Y9ir;$Rj@f++_=Aw)2s>UPZ;FJSXy`gz`3Py;nnhbc`Y1V`m|=~5SMep zG(eGB=eU2p6qv&DPs36|&wp})m(LTwA=4RjnSrABhMxjDdVYf-n&`qt^*A?|@r`)R ze5IL(m#7MyJ6H-xx#C#(8d10s#T4<)q9AFp78Y|sB8ZMsQ&HtFqEkh-7h5QDT<7i4 zRGeh!HHjiwP<+|D&T^iGLfv2?znPs%`fHvC>FVZAQ!N(U?3;*Gjwg+nxWTgz9e#6Q}2S<#Izy z7wHcY?9L>jRS_WrfMh+stZ?rxce{$^QS;zXi`uFrre^{si;45hJ|Dd*8pWF3N; zMbvaNvaM6pWc&w#vg zUJz006>5bR87#hQ9wRCjqL_nxzpx$sSe^Br0a~z{Efz@W-rXk+Z;bF#X%VvJ*s={d^BN#P?t~~gRP9kwf(R6?f+sXfY9++zFb>tY z*Gg$5>`aK8dTaKy@tpDk!(rO-g32m9MvWV`o1;RD@3-5Eg@C%cO&e?H6TR1^sNueu z62YUE!r80|P_k!<>n3U57%*3&T=F!J6+!xGWu9bPRtL>#@=@7%%cmi z_6SB3W`wRxB`>MEzr100Q8eoK9lqR4QApXkWLV4yt`}Ul+{3MxjMN3YWop|~E_Kym z6EktGPVRSwq!pJdaG?Ir*W$gXj6cL#NCr|AdTGL2rK>}xL&{g$3#$&w9b4X5jsj!nYbZL776o+;e`2N3 zwLGmxdoOz0l_X{XN`r_&Hmr4UcpAeNAw`_m4pD)PI&KA{7o|B*9|1b3azaT%8 zp9#4j7sUR*`XBk3%+(_6srgp(`K(Jwt&CkF^%(~?i(LZANrcDUpr-udf0%Ef&VNNj zp<8%<0#i68cVIcp=?z+?&yuR9nHucL7xU-tXarGoo4MseF-kMjljo<4&eWlZVLL@< zyJqU0S;MGlv4bw%^uUmUvtoBgcR_++fvT_Sv~-h9bNip9wz%<=q2|YQ^jl0kk}}~%v@91-KJ;yg20O**DZ|m zhdzC5%vl||38d(+JWSNf%mzW|TAJ&333AI;eUKGm=~Ewvh!6B5ZKZ1cBU3~{6-dr9WxZ=w+8OO$$5SF&Yg z&S7{hQ@lvBI%TO*&n(6)m3Vp%)`^#gw%*t_vP_RR-04R*PXu2^!)R@G13ZSK0ks=+ z1z5;x**pTZIsuE`eVS`5P<;_hy>aW1`!2CMVdn)z_lg3KnD3gcGV01ni?DF0mA!%x zjpa+pbIx0!0-|8aUVR^zQbc3cRV?Yx{JEfz0C;C1Q;R}LsY+FDZc16M7A;i z9aU1vY}AT3nsv^p+ZxfmAZ8&Vj=sA|OvO9on7gkHjOCzn?lwhXG}$kD&dpi+CNd)T zs(s%WiF6)1>=qhg7cd7@U!Aj60xY50m!M@ZE}H-ao?>;m%$8D>1k6)b`cjyIfoduU z!ikyUg8b07Kp0In7&?v}{D=GEsBJx3*2xX4j28*P#J6=tmNcV9qAp55%BC8|999o> zjxEBdFfW@>V=03!tr%1e7`+MeDR$JF=vgMkOpzPY@V+q)y1<{^S`8>Gnkh0?=~R8o z^QHH_$M|XC-k&~UT@bcdgG;@4w2r!I|E4^_ky~MeRaGYsZ=$g}YET$!b&7PR1WKzz zpT=<1eAyk55+{#l*y~R^h0f~LBaC)Nb6yQuZVtWAd-ka~WBR_@*5zCfYew}wb$dM& z-RCPn=MhX6$H$HJ3a?{7K2nK89lg-m@`(|DNyV~EBW4@59MsQBwE8!Pmb;cY_kIjC zXg8)Em?qKQfF#byvUXFqHNRDiFv`G(#?TGRwxncf?F?+x3KwwuGus+~Ad#9flB3Q> z9wDrf`;vEGKmc=NG4H_LgP-#r*jx@qmMNTGTURvSXV+FHGXHC{rCnt~b%y zteP1X@=$y|H~bc^pLJvJ1dtnWDe#Is$wkSYxDS2Pj?;GTGJOY;aNTiBb7Kx1{9EN^ zNArEsKJeC|vG9KMs4lN~^S>|Nd^msi=Ka;#+n40Z{D60l)hRFrFthefSp)@=t=|bI-Ya&Xl zYjQolQif{4@h8>dur}a7+lk0cXjaU6v~Obomm;gp3RrDHx6S}G0CiE6O?FpL9hvL( zPkej8+NjSCSpZY}DPs0-^s5*sq&@O@5FoSWvPquUmg1JDPL+lirs&3>l??nJFaNAo zVK7k@jJ$7Iv92~R$@4=dxM4J9{~vpA|J}H4<&E#p{VQ;$-*Y_ETC!)FHr<@v_BggD z_0xIrT28Wil5PefAqh1}umot2`sDfFzX$gMBq)&}D9Q8EkUu0IkpzJY;NFY-_UTF^ z84|JBtEg_~d798Gh($DqIM_*4+D>WBLqyyY7zF}-d2@ZP_YHt`*;1oEOpwlaI}3Cf zFxyuP5+k~0Y^5470}IRq@pHN@%^@=ubt7H&)ohSj58~PKWkI9vG9{jZs3E0L1B#$- zKu_AF#6ryJoF`nZ2?ZZFmepgGMIZX#ZLBdB>j))_L{KbbM^1@a6{1|O$n)N^_@6>8 z)*Y&_s$d|3*GsqQnzT{U`^G`82r>I zl-q~jj~mcY6m_c?=MKs(E!9dCdT#)0gYF;#qwWxbICi#5T)^DD(ODylFh3x{sYa#u zqosRW48WHS9|hS9OT7x=9g)wsu3s4PdA+o;o@T&soQj*jT<~*Z3datC_T7T1Zebyx z4wKj)keciUN=-(#^DobKwrcnWjOQ26a{`)=Xm>?m(pt~XL^YhBftIz6{$E!+^NeUH z4+~gyx;LSkCLTE*EfIW_n1+nXW-OrMF+4cY+9>*1yHBHJR44<+F)EaOz5B&+_EDi6 z70OYe9NEsOP=5M6Cq$uSh1yZ2jB2$E(IvKYOTSiS3V$C`skDxb*&WZn1N&p(xGvDg zVKYu$RTFqok0u$ia_-oAKSE=zh);sfcWgcDk!C%TtZiKVN^x)B!{l7AjC&;cwu5X7 zC||JwBwoi!t;7Qn2^RG$)kG{NLbrEf z{xKJ2))~Wy&UBl)4&e_4lewuQ4iHwei?2ya-vbpO#qDhO?*MAjl1T-5M(!7=B#=5n zWleHL)4V$l+QeO&X8-h`I04hw^u4AenY5Dj8I0guA%l0siSf$^DDz@BV#y+6D3VNxIEFTdos+s}p+5A;&K8TaO35wFm501+dl9?0xId z!(g}CSN{;mEzi9Ap|&>WuLWnzWR!PdZMAgVZ3PJ3Xtonos}HryB5LIoc^x#ZV^hk# zVAV1~_*9ee-%jG05(Ql{t%I0#FI$KkhGKCjJgy!ru0EB#7lUhe{4G<+xl>En8GEZ& zy$HqJ+7WNd!PM#;+z_m-2A)=Divw`Bnpj$$E%sn+^(V7>)XIb5YW4P_4^Qh+u(bBZ z(ds`}^x$Vb5_Z<1akF}-F90uVN31NbHDgDdEMMx&ju=^91ixMIv2cRAm+mguSpE1{ z`{l2AFt0lCu69gH!QuDmHprAhwHn)K+ffczrzllks)p_f<_BnCT77N{O|+%OZB590 zycNk>OAaobN5wsgbw6YgPOiHZXK(nkTnKM zLrS9+W?7qZT*VuZ;O!LB~tDH@xdnXnWKY@ z(NGm(5I*lw^@?4l&JPWh#9OaIUo8vtYC!~6bv-r(XtCG_m(@ewCg*GWb8?#X$p>WcB-jsvtzTV))5*R82iT1Nqh#dj!ZP@r6_QJ(Sw_s|otIEn=xR!7N&NII zrT-Kex%_%&y7lZM5p7RASW6+V-tBQ6(SMc&(|SpeqEkJ+sodz_$puvubsXS8zO?;H zAZqnx4tQ1Yile0Lp-)7>_Lr|kJV2>%Ua~CvFS>{4_&Z5;Yvhf{m_J_UhS8YloqLRb zeqV6Kyxb^A1nDL2_tb3+$i83)C6+uIw1} zpJK7qedwZ}Q68rI0~OU;Pkp(eqt>F_gD_ChtMncrTHDtys=!( zU96u5$6HiaJb***Y(+IAC25%`o_mV^5WD2{0)vvpcJ=6kCvYip!LUwI{FQ~M>0N=a zTw5D^jlhS7yH+@edXe|H_!P~QS@oK65@9y9Wa`<}YKMrM0%X{m8x2-@Dvf8$21=gl zdnaI9eKrN2*{k=gSh1xr*%v#NqCNGOepzfeuJ?g&Z?5)|)xJ2jZCS0KZk5$=W5$Ym z9QGBsJ!@GD(ous$lHcwKSc4 zcrBtfoHT~oqf_)A*Pz8iivjXs7Ud9WsHliIV!?|rAo+yu+tu{8l`(1j@U z+ANxDiKz&Pa8gQEO!3p73UT`c9d!1Fb(B8pChZMdv{BalT=|3j3asd6KgwPL^G`(3&yX7$Hy&Asg_0xI*3RgrGR@Wr2-3vv1GJ#XX#Xn&vNxj#s5 z?m^Bl9dCC%q~^>_=2Tch|{pZ`9Wiv&iJFiz{)DJ#; z{a_D=uU$5X$YFBL>Oe8vKp%5Aeq_IGZ{?e&C8;V5($od2pAqvDLtJx_0}^;A2@uo9 z7hV;zcMT68=A?CAYO<_U|1#&QVmN+xAX5m8jEMhQvxy8i9SC&?`fOTPhwZURMatsF zRQSrLr>Ccgefs5>UlLIeN@5oAbn`)P!`Q;_pFf|H|2cd83c??AB@sIvzp5W-H0(jR z)Yt!+y?s-o$Ti=DQ)5NWM&Pi&n}7J?iz&I_1z;I)1N3AgGHu9kCKRE7s0nJ4|7^QL zXaxm`o0ulRmn&tkcJ%5^c%(1|5m~3`!J{HJ>JR%+sev%yW0LSYMi%m$LX>%hbZP=* zy=|IgNbE$t79~?3Fb=8LKH1ny@brd8XnGTL$h$x6uv@m;K|L)TP z0u1&WRjzZr#89-`;%c;!OF|8$K3++uyCw)4RNI5KdC)>kX^&WL>DWvDKI;4_NgJ$*xoAj`Il}%9f!i(L7So#7M(r|N4Bzc%6L2~l+yh^yU z0Yq1sV#R_+?3rs}VEKxrtnlE&+h9nW^R}W{Y*3jaRsbVm?-eWbT%6CYNL=uHR!H*n zYhUWRskPGS`42Krz?Z@UmEm#Anloq1)!oKX;t>0esFj6fs@EGNXaXx-i?T32FI#Xa z5`{iLW0TWbc?v*%b?Cg}M#6xNE?!oJwO& z1Xp<K^b z_d|oR5k{tpc z==Py^E*qrW>=EcS`sfa9Y+?u;IomA2LS1d#ZN^wD1)vY$kDj_M&|wjkP>T-uaT6C4 zO$+@q<6XJ|2@=rHUDeuHJY)~Fh|jJ&^T(PhS^d1Ll-i$({+-T6sca5|$Kkc@9s8JG zur>pmsXV{w$?`6VRxg*&d`D!D6x+e6hL$pKtD{qGbgF$GPPHp^s%c4@a0x|q8vMqV zq4HblhWafoYEFmh?Pl7Sqlx!vGQw=ydKNb-gXhA4l^@6h0&VpiU|Pm}RHu+2em*(} z9S>wKddzxwfwDuhOrT_x#|X+VkMR_**DGk2IgOaN8drT)f;GUr98M?#)t#(*8uIjJ z1*W>HC>%rfQ4<1lzoKZh8d^{s8@^NSq#RuAbViykR9EKJ^F{M zV+3vQ+I@IjSz?>+!-i@RvJ3l_k4*(5q}3o5q^V{EG#lbRbew}B&Z7YLp`%#@IcG!M zhmL*`#3q6?kcO5K(ClaJK6HfUweciwyAlJ4;M%u=&R876sQc5pUa#J2#*-MbSBr-G z@aSkdJ;K0``oK`C#pI8KWs!(Oz0|sog;bzc)ut%dd79e(4suIk{|<`EWa|{Y+)*j2 zRiUclkr{ARBr#YgvMorG@RURGfEXALlS-1O_~SIn%MhMno=#aRiuG~TjDEe*u6g2F z2)`^CiOQm2nM&4=sB=}6>q};@ujZuG?lLf+Rx%xUu}Mn`p&@ZxnTWQZ-IIG-$o>(N z;c8aQ@3|B~%SB7X_1l2ONp#FtZ0Auf>f6%{} z7yD!}rYz=VI-rmSJnvHE0gBX`; zEf7u75C|b+rs-`P!ag<3zvw~@*sD)XBhR#+_!`uNN1r|tTm*pslfBY9j@KF0CndvW zW3WWrNcgkew2>-Gz&X>DlUa~NL^KH~mq(gf3@n~YyH~9vP%k0s{n-}`ieSw+d( z6a*Qi_IN2vIBb{^p#JM^Q7pi{U#Xk2s))X|88e=Q??zj|x|)PCGl6IyWcz%Qy3URo zPQP%o5pO}#?Md}`Q@lBqp`D<(&f(10gxWZ+iu+qf@oi?d|6`3k*26bv3k+E4-Zi={IA5D|n3yF@K?Vz}%JyUQOe*g6;`}iq-%50}cDrX5 zUlIDOksT`W_?j^mV}4A+s^*I_lgDKK%!UYdH!Kz;(+7kKy#cLA1=9qEIyQw!iOgs& zR|4QABbw8QD-VuiL8CkUD){W6d&eYtX2P4@X=|qVs8u1#Ww-uDtGA7M3s&zJ($-oYm#Te|hO z>K&@ZK}L9{&*5=N6BGp4`OYTKXII!PU#`^Tfq`ciWqQqPY$y3dZY^8Q zD%2ybs14eq1Q!5H=PZtE>=`trJY=k}yW;x0^JBuNY-<1IZPGSl#efn^jIpm$K48Ja z5FC56w@pna+wQyn&;!?R{%P_b4%)$=-f4+b=m!EILd+X(Tz3WD_U>B5vz$d!gU|C0 z55|OLs_3b(buD5Cyd3*nu)o{r37E3VZW~d>vfhAYn+=-@WpJz);GWVnc?#A-EPilI zln}sofERk(Jj)1s&m~ksLE6I9Pz*M&6cqhQy%KEU8~;KTW(oZ!6Y4kB(zii`pm7L7 z?*+HGR+_eUS;H>>Xh8C)gh zYC&WfX`z@GY+}`+zQy0hP)*UGT$m_!{iPMf$gVu@9X!kp&CHhEjOB*0i!7>DA(+^n zf;MkJ14`B=Y6z_b#;=pHh)GF{^)Xr(=WjS9e-u@S5gx*~~)323e+f!E9Xvy}%8j%M2HGljQw|Y0a~jOvySp6Xls^j7EDgmKzXu zFb%z}Qe&)WKN#R7N@3)ZmcD(1AL61zJh;~)4)f=2K)2N|*U1-{+#|ufg%1R|9a3mT zZDQP#6}{ix(p+n=lL~1Dx-@~$O`eCiL;@lT?KRNT7Lu=OFU_N7!ET|A%0Jb+hSi(6 zjVdxI0G`TTjB6J375Pm=!7<8kb)v_FoTv7PKQ$9#YezL74{=zNjSqrm)_zJgi9=z| zM63C0O}s$hd6P1!7jz+#gcU$8bNUZk28}8vfVhILuj9)6Qyo<@_7K#j)z7HJYLYfC z0nA=DD$#y3%yC%v3jva$jk9cx#Jq484H|rcT3za4=_5}@=ZO~R&f=^3+mw4@G!;%u zH*8@nHSSN%_JdljWpAT>6uyndr&S^C%(z*r*aVop;BnTo{8*z5$np>gpJpOMwujG#DQ;*p`UDq1l~qzGE>gz!u%nj|$=w^;xJG*077auv}q z7Vc!<8w+TjP-CK6lv#wKj$EyW1nF%Z6u_z>(Mrb`=t-!p3t=qPE+nwbtx<}uEb=TI zlub3kmm(~paZaPV2U^4u3LH?y5wYzUwKiA5%8ri0wrD-gJloVW-9bOC?}csCO&#@F zrTa=&VSv>{2y3)dBBhE)U;*R{$GetRC>1uAB5aF-Dgw%qnJlv>+qKoeb-zhSXLaG! zI5Y^GrHW2i3eo6ero*=wLIR%jf(35QR3GXnz~C-yO^C{6>U4N(BEz(V-;nRYhwik5 z=G7z-50a<{QCYoE`wN?{h2ci>c;h%=+STf%p30~XN4#ngnmg8e{edNk{=bi7_n5vt zqvy*Ev-`QUrb&}r^E56*o`)(*daXa(96f}A6!%O*Fpy39@xe2=*OFzdpb8wHrh+G5 z5W3aX##Y#w5biQSC$pW>ft=czcGCiw6ZHzPn>TQ}pfJ=k@~we@ujEUWm)&6>2vdU4 zhJf|~^xPN|!IV5bOY)WT(zqXut?mq0vEvL`A;nacvRP{-f`Enty11*k ze+-QilQBZY)$kH>jkm)yW66%$v1GRlS@NkSXEd(^B%_io-t!7G+VRJ^ITqT?BI?J( z-@L>EYD3WWxH$*2FDOZZwFz0G#KzvFZSYOY*3{LGy*)8Gc-C<+m;w<*Y7+K7Vk|bD|NPj^pvSQUKv+$R#(K6d zn#z21&JyyEk`-&B^+KB!5*ou3iA?g?6gJDsmL8)+6N z1X2Vqxr&Ug#)`qB4IMVrt){QDHy7dOr~zS&WXn=$=DlqY!7UObU)2V1iYjchveuJ3 zwhlKA5)*iBpn`3)m~qg6HyhAoe@WlwHFR;}6$D@9UR_XDI5&Vr69yFIURYw?ObKL= zYqE*(!INF=UU=C#+k*_%v^SD~@7TJhqVx8JRjouOS3I}yy|B;CY9Qaygb!Io?g;U) zdzBrNH$v(ET|x%Qz;&sw3&G?Yq2T=wGD$f6J*LR`_c-{~Fv|-nZ4B&3-IpqsvYLD@ z$yG)~VW_*q7BHQ}?pgOj?F}E;VoU$bgo~mvg9qzsP|ZX4FD=X@@>S=}Rj zPWng5?`gs?pOI!2=9^e->7U`;0zqE2N>JtYJ-K7+C&!z7DY(TaSJ{)g{%3;+RXQBR zG(3TOf#r*d$<*3ZfrqKKk9ls|Hw4w#Y0ndi?v&xS{>>IzfU<(_Pm_rksr3-~<*3cUv)&Ew$08lgJ#nEzO*5gD|`JTXnKg zP@AGK3DaBQ$$NXc zV^a@Zsro3T+!!z=ngfA!_ocFq;_nOHEKY53AH&Ru&9=aB3;6QN7V zHN$Ny6;x2TL7(5YBQjKjJtl) z<{fg#p1|yQ9tikW?<0%OA{I?R)~IQIe97kpBj+ocWh`+DF{5j&v(Hp18m&+kP@?%( z!qkx@D5RU$L1%4zrHPvoLgRS7X$4}_*YGT5|cdDh^rxb17m+Sh$$UI0#Q z^Zgoi&#(a?4IXDfIKi|2McWOQORmz&XUQ{1E;I9jw_V=QZl(q-b0r=~EXV_cKqRfF zLS}&sLCkSR4@r5ENb^FZ+(yaUHL-UA-lX(GA~k0?DKkC2HPXz>bE0UmWJ|l@|IWz9GE>qkw8KC> zk^J9(|0W!4?WlgIYc?N5k;D%??l1Bt$y43XGv7!8OuK`SVEx9+hX)=HAO<|%B!WHq zY9bI#1;S&NDZRN1zn=CKT6;z=IhN#uwgH|co%5`&?^sVjpBG5)D{JVNOkvS5DEc7O zib$T8iJ}=3rA*e|<@6yZp^*LTFdyoK38k zYN|6#9f;RsNbNZ$C6E8yXB8Wy^7I-nc5xRzHvIbn3igJ<2AO9!EE?uVqhvXjif(I0 zhXupzStCttDLeZCV_f%ex8k1LL{3taS=K#iL8=*5R5I0Pj%(^?^*fq-xO>Fh0{;_> z3GHDSruHnE{K=?!S)-MOC@R8OxFJ9Pwc)k!ZB65XMLpGzgI2%k(YN1TT?{3Ye#a_; z@OP6`0Ukea^YDL^?hy=1{8z*=ZS6P8-d(heSZy-B;PNgEhHLA!=GOY={F0bo15YmP zcEUbGVRy5lt{A^3rp+>9wJRm&@@^k!uiZ_Lo96|KJW)f#c-Pxnl;pt_7&qsXTb_SK zE;CsctOfuGUF*vXpvw+u&OGWea>$Z7e&<(2x>chLY|5OdKA%&B+M2TGaP1( zr-Gg{+lOL%dJV;ysO&UE6bP1QpysF3qbcNM98Hg`B9_=-NpGV@ zHr1VR@tc|pZ%W8po80Qx*!WhwW))^xLs+(ap^vwK-Kw0soi$;bOMY)^Iqq$XuDcUF z#m?rm$H1Fc`}YHP<qoj6HVd$k2PvF{Jm^YbEIp};%?5ZeY(j6^MXw(^qB=u7*y}JvbNiIo7JWsYniilJbXJ@Z-nwW zx>#qkU_$58)T~XCbqxe)tRr-Z$S{TRl;j1!=LuW-41p`2EuAV_Zf!18udZpxf35#j z*SY9y81~1Tx3>V?3;yE04?4Odo+UxZWHJGh($l|dojFdPk?)!LKBPCOf=7xxtV9Kq z3S;Y^QP`@8_h#Nc(Jg5G6UAer9j@gc0_&mocrX2W2M%qm?Uh#OtjK8yRFdWMl*o0O z@a(PzF-?gQGb5HrgYeA1?uA*wA9p72BvXO&7dCR40D_ZkI=lg z$M|duk1@K#hR-+rSUr|^s&?>W`P(!-8|=sO`>#GgDE^+326ptnitmbNTyd&IG01Pw zc&jp@lv0TlcNmy4d2ii{H(<=WT5LC1=Y_%<2xxwl=~GEx;pS%i^D)WTgXt5u@00&9 zW<@~zNdELyFnoCpRcH}kjjbpWS;nG1(A=;o`A0b0!<))pQp%oaFPfQ=se)!w&s)=9a%7yGN93uBa<6~y;5jf!nYro;_VX5} z<5iUd)c=+-JV}VE0X;252v8X{qDsdPQ8==Ve?J~TarGgcX4lZ zW_WCIBZe4iQ)n$~hT*lEmzqrui?4?v#I@82*VD$>*v~vR)~2^1V+vk)?n{vFz0Kq> z(*ytZZ>>T6&Ea9*Ru9A94sy`v|LE%KL<_sldoGc8H5B+ZUUS2_p(-Y=3H-vCBy{m+ z_S37guP|Zl}CS^$Nwar!cQCHSp2H%Mp$F6KXZ!1L<#^G9DH6?^> z6{VX0dA60wV6z2{+G-=&I2xKdXj3FHu_0eTO;o_4ABs&Fst>Rzitvhs>I` z7n@G|Yk12->sU*0}|Qv!RhnP}21G7_-L$o5Kmbcr5cUF^IuMWG%f$ca>zdQ&O`L%`(ha z9J^RXM5Jh9p)HwuV9c*go}Qkf?eX->FTW%J5EiqDr<;useCqelpHIpEoW1si^b-S@ zE3sk~_6R1#ZCDvsmM7c7f$${YwM$G)r+tM<&qZ*r?f4X zmNVFbBYe5C0OWd&CA_DSJeX6^E^KSwV3;L*r;;QvgkzHMJ4O}~pvV2WB({cf!K7CC zP4#O0P`$b&^A;gGtTQqI)3)g$GMm|g85wjf8%U14pl34#IQbgiyslcsX)pz3YQ{qo zku71?di*284K`wNmBqWxdptJbm{Ac%XWtg3R9}}_oCFk(ZH)HG`Bn=U+iOxYESz;v z{b4;v@1{^0SJmykt08T-KNircF`euEL^DE5B_>dY-UeGw$&VR_A7K_aI?BZC6B7f1rGdpPH+pv4-M7dv;FSs^|_ zAcdhF)r%7U<&K@Ji&2f0x~k5z@vdp z18_m9uDF(GD1AkAcJ)h{ZXm z#XDvgl@E<`Xp7L`P(03E5jl6jop>sYGKIdZ~bnb)ExyOX< zg46kUNS%*`)%h4`osWpu`6!5;kAm5`2WsbzxSe~f_1&;LcR}wQir={tg6B>cp1aUs z2OQ6Rkvw;tzz8vM@#;Ck+`F9yG`AG&8)+&EsXXI{Gq zce0uhG#!~L)#g;$aWr*~v4Jh?Rna)whfUkS?JjJ&@!>SoQoAVGF^NTXq>Ond7cmxp zD)aCoz7xW)h3WQWJeAM%^39Gyxih_|JfZW%p7SBp32l6kd~AcO+CCQAZ&_7n$OP~j zs88X3u%7c!MpJdJ)&BU#m>-+#Vt9X*Fs7P+vO=^OeTFQxKHqD7^@S7HS7V*sz{%s@ zxQ_se$*I~^2oVgsEx?P7l(XnXFmJDYKu-896Kv(rI64a5d+?e-DGs5C%_ zs;HGg2BD7lDNKMfhycTkTK)j4DoAQjn?V6pgIcVDuX+NkcCaLO1YWSJK)T}SUYa@zlP`jRpuMjYBdxeI8Z4E0Z zBy95vKZoJfj1G0`juwrR*&vUwGbL~u%^l`BC9jRZ8@PC=)`6u+N>!CW-C1(t1p~7N zM66_Oo2;ej;9(n)7%qg(tHTAvP>cHzYkS~v2(KM2iJ{i@9nFBfMI-`8_kl}I;5D@1 zL}0TaWhwCbaHdB4s)aqN%FH!1`M?2+f@mHMhaanMWBIx;;XlmmHE7}QV-2^#^=H>t z1FZ?pn;rH9=ZVkOqHu*41t)-sT-3w*ZixC5e4^j|@U+#nms(WTaU8*7!<|qiOv*wn zd~0yfaHo*T9|lJAhzBxgzo*#I=GElxH&*%?t14P!!h4OEu;pdqufQ2cXqE|u!7#^Y ziXY@iBvlx(o0nZU*=K`v$6aHsYWc8N(28lpQb|TDmeRp`mcCx(6f06m z#A0~xhnunXUxv{}LtI~jWTAx<@b%7Uhu{XNdz2O`UE z(H|8l2P?`0!#Vc%MdP)fMW)3PW34NLX31xXIksxg(8HC_5@W3^!`eJRYTH(uL8)$o zn(VX0z+4S#vELHoT2}@&?XkoJ=gV!k8I(Ua*dpb(#E>Y+tz2KfEK7`|-dL&Gu*wWK zbhs7XtZAmMy&F*1V)HW2oXRL8iUz>e(b05zG-P%v>;iLaE7fB1N5Zm5#5MN z)Jl4718^-`z$PnO&+r|C;~O%-okXk6$bPxE!d$D#Rn`B1DqEXZ1xaG`g`SwmB*3GCOqavW)U6d*>m6270r_!)Gi@#Cr&g1I5G`LB9zIxuD*Vw#rsuU}- zugL#C{qeUyPoDkx>C+!hC;$Gp-#-0u3V;0O*`J^NZ2$4wXV0EK{o${#zq!4>{2PDv z^AA~>-r>J~e)>?+1!k&3Q2kVb$vTrwh8qD*2^ zu)iCBbST5e@!vYVX+#YHiK`(1nb7)x0gP9^j2oCFS{^hJ?Dhxc_C=DV$$jP@qpPjCG4x> zwn@3<86FCtWvwM9Yf=bBRqzVh9-_Od>?id{0$W6tO%-c^XA!154js6&5yDMNB8e@u z=O0h?`8P7VUCS5%8&iTL;c)|6IxXB@H9qm;3$hYrA*UC^d`e1?=kO+)lE;vr5ds;D~Yxq*h zLd)EJCTaG85B96_y$$`N6!WzTgX%EPHcRvUm-%rEKCrXj4y>So53JDxJ6;OWRhES) zy+sKcV}Ymb3E6rSprW<@(E=iYdD}Zv&`$3ZJsQA88#aZVkr2>-u>*pix5Wd&1{Z9} zr7DI{1Yc+_PE#sL6eKMZ#q*@1j&~3@wlO0mUGE5WOz#wtWf0Oer`*qcpH_WfnF5>v zh@fXVEoeplNTey1&(H}&Xw)Nx3^tmUiv>+;Dmbw(`z}t!GMagPr%e=Hc$|ob458W> zS@L_9ZLCTXU^4oeMt3ZW$+yxHCthbXHNdxdA?~?eoGjjGyC+f>P19JH*_KX6wQ65V zm2v=Jq<*-M|420b7yLeAA-bq;bKgPSd!d`QhGD1E*%hXMukrh0Wza;k75{n6B9m+^ z6jSK_(E*L^*6Dn}%T9acY%vc*vNfvtQO}QmjQu9w^pX zG{|?iJpYPZX0il`9jZ1y)2pWeD+$BFBbSD_evR@2pyBNcEnCd}=WH~guEYb01$lsw zgL{2;1Fycc))Z$l7lrVXIudDKh?Gk!Rt%im-WZ%brON@!5gxCM4zM@zCsU~`%)&ir zxBYr09wtgmO35bN8~YYC;5)X4K)~Q3n->~qKPAA(3%_e+$@|f1Cu8e8CRkRha)<&Z=e*6M3NY+7_lJdEMHYk4o1B`@c08lP67|eAx@-R z2(L?G1F~6&q`9r&7u;3O!YSrG74<%mWprRj_D#S+KeNx;0e&6)^$$j-ew_8FV5%nt zWD{!LyaMNQ$I(bjwPKm#5n6U-f54&3y2#Z6PZcqX0sE%lF_Wa?PxT`7Hkt9{Lr^!= zeG}rq9p<|I&`#N*?${tl>mWYYYlz#;xj5*&cmxN=27`oOwiqOhQ{Iyw{YkLM{v#Ra zRh@UOP~G)JCL@}#tG7X?GV^TnU^X*9I{3^z$B-_J@V{1zhh!q|b`*^G;LQ_}v+5DH zcweEtq>4SzaG7nMx8#G-(Ks%cl$L{Cn zyl4mI3fPdrTvzQ7)6hZ`PKn=ApIgm&@*_e+Z>dC|4>a^}m8sU49J`!NF~GO95rzIObAAdo;tY5%Xq}m1T@NuFIPX}A zWfilTp%UDqQi_Q80s+HX04rB5#$pg-`;QKko$!I79@Qd+?d+Q;sGoH>QkQ3zHhT{S z*HvG(e~YW=jgx?92URW8A$}s8Yi1_fs+}u}r=V)O`yyzWn?1E-wp3_#R9JvvuXmv( z1O3XIfo^lld*~UIcRnx|yEEIoKabqsuA5_i(CqO6Ipc0NxVO!aO!4g;@ZRa_$`9|! z{Py#|W>)xNP)T;hA_;J`4b12c%+wA~>IllpK2$FDfUw)JyzAg>>j9N0y*bt$dD8*S z9#k0P^#~#vuOWgNvzT|-p(@)Xt5;|xi`BwpxMe0Fes-un9+sh7W8Kg2V@=`ESVrI({x~dX`MOh*KUTx z9}F?34>DvN)$g>LXKdZ5UiHGnW7Z|rmC!u!kx$1YY@PPo5b&#_Q`Km*)4iQ-I8m#g zt)gvwq}@)`(RX?X-{~QvNk5(!FaBkE@noa6rZvU^^8htX%tH_|fcsYa@9g@DL=u(-q`;-u3uQI{BQDloy34j5o&}hPO?KJkjSwR^ZX7 zF!2d9rpyw7<);Lp873O_*$Ql6C=B2Fisx)a@3|-oY*`8N#N2wKW#wA<%X?ePbdC8B z)RZlAR;`8rBXcaQVQvCA|69#f(`C!$KeEPnFLNzxzw@S;HMfBKMpJXVpTVtN^+r?W z05dpkGReo#9sPP^pM~}ptv6P_|Aa^DtBc_=1NfemKC+4z(7%(fsqA5cuW3e?Xk>Ug zyZ-w1Grdy4dUSQM>4A2&E&RZn{nHvZNH9qHZoT;kWH>9l4WvIPpzif<4A%FB3Cu;y4R4#>HlYWaoQqVm<90Z<#&kS6w=pOZ z^Z4*BtCz!JZ9-R!#v~C-^7Q+&SI=6yisEsW(L7&I$&VTN{_ItwJuXckSnPPvK?%4n zM9S2Pm2G14Pyb0?OkbQ%Pj#C&*7{w)LhfHopFf|X)t%18J)3T`Ip3l0fz5j;bOF{Z z*$9)Z$`gU2RrltB%}H04CcmW8XC0i1_C?s*rN7+XUXz>_G-Zkv=mzk0p}!l@6-Hy` zOR{V9<&{CdG;iTXHpX=t$0&lJqXm<>$fU2>PS|17ICEz#DhsaG$P5R#^gs_ps8`+2 z#5OcBY!!#n@0r>d75O%kOld(nPiS`M7vq8FLj6FbZVQ@8tvT49?l9OHv^1%jszz;w z+joxLn#9fFlIw&~$w*4?w5HR;yPc%(QZn@BVU-Q7aYOA4F6~U9KM5>AJ&n!_@gQLt z3X%vKyPMOxi12hzbFD1V{-6*{*GJ71WU!XU7XQ#RU@7ZkPWDnJoPm^T#x!-5fnmyW|OaT^H-Q5>~Y7wSDz}`4{VGsUO@LBWANpOsfJC7x+p@2aZ}2f_!I!V8x3c1P_wXGD zs=q2vZC#Z81w$1rmdx*IecXV@+d9$_VdoAS=onU6|I_wnK1!|pusZnzC>>rOzXiau z8LU(*$9-@NMXY5Y^ykA)gMDXa;{(ECULgO-U#Y zHADhIIda?59rM-%z-weX|JrP43tb~%JimCJ6VQCb^A#&*C0G7bL3cA)TXcP7!d8fc zW`Hwko$MYLpl>x}1#129^XUtSsAp-e*2m- ze>OZPL@x(5>#S5Oe^6y}JIp|{A~^g zQJz5Aj`}&#@AVAOpU)d3?_T!->pd!qdv;wHdN>BW%EU7~H^a0znDn^8Dg2 zm%h=`ES4*J$By+2dDRgjikKBs_nd+sY93HLEq5JVf(Wd?b#?Lak0y6)J?oLZJ#yDt z|FxJqtku6)+&+@1-9gL-GX8$?`YDUnN5~oqq+6J<94;t94EC>v5Fr_gDAgw%{em$_ zDDEN@@%sO{hJb54Q-Oi%bdO8WliNmD4IACGu8Vf7qP^2Z&ZIO z#MP8%npSIr5ZsnZfaVFCBgGrh{xQoFvDV9Yd$IasVSIrux+}cngs`PeMP;EC4pR7%F*!?Ep;AZ=DzMBf&4j!u6IN6$ zxAB8M9W|j6qt#&mw>&VHaQiiz90X9pX2|*}6Ij;Gs?d_}&Qd9Dw>$M!yLNQb6A##F zuf$53lpf5g#-RVw%&0c6+Ooo6y@SK7>^1&Z3;0MPkyJ&?t`YZ+#B#rHSZ0cT2My}& z)@z%qc$zXrW2)$MSv}J|ofh@28X|e11|O2@^KO}o43GaDJX=~o@&$R<_LX)szkJ^)aMj>n;8HHsQQmO5(=xE8$YX8SIX&Tc)t%>)TD>2UEI<@|;tx z&-MuOo0=`z(*e!agkqGh&Gsp?m=y_t=cBl66SBr^09G=b-JBJA)3JC;Ofj5mX*k&Ry)LpJgJOC|0CAgVbkG(l)iNmeI8;H{_{WFyZPvca)R z6weN+q1i?Q^fUXYtuEq4IWDOa?Jk?FMQNn2gkYniO5UULXQ4FKQR%#>Wx zXk~vztd6!mQzFYWrA2M(({Gy{K0p)gQ}6=L2yLSrd{XEXe48hT?ehZCtzQ&&R&ERt zg?kLXbs2t$Uc)E4YcVuBePvVei^hW5S!R~mVOEaQPVjzqM*p}3#Ygu0H&oyW0<7qfug9%r1poKp1 zGJ}WgVLGj?Tgke%Z#g+n@c1DmchJ7yZ^!#9NIZ_osCTD-I7O^|w_D?V`>x-0;O%?7 zZ+B0OLcZ~1>t^rY%KlBA2sJ=@_+x|um0^{?21;jQ1T=y zV))ObPC6{ggvsferpdnFI6XZ*{qpy}$N&EU|9|@83;h4-^Dn;m;`HC1fANPee}DSL zm%sn=^xsZjoc{jw%YP%MhZ&;(Dy5>uznz}4f=9XMIjQJk;kl24Ij#QxnOwiFd?^Ke zn4&+Wl&mmyVZgOguZaoPR})3+Z}+FuFJDYgEy!-$Gn1%@PYk3j&z93j6ii6{m7m&H z-mfN;$**d+|5t?OoV`~pgOq~`#P|8H?syh|Mb1m9M0#U531qQ?(dJjyfT`ID%`$<3 zG=|kg=-5nHFv@ArKe?{6K!X|CcsbWh&0m6SzNDIl~4$`LN1a7p4m(i-PXiV?*2Kp6s=fVB{zti zWoOq{-~DdZ_<2K7XpYB(SV*n1i6Z9N63bjA`hcD-2@DEP&FwmnwF_52wV&>t#oCai zP1mUIy-_M+C)e6gKp!w*WO!WUvNrp-S@^{Q&^kciP6_(=rG+a4Z)O3Ss#s?HpHc%%VXDMSiwvMkO8&y*7` zv3VXT=P6eblr1Kq?)Q|O1LuP1L{!?Qz$O+JtmJ?0buqkkv?&ibW z;RJ^Hb<7tATiFyNv8Z(O#sbK`g6lG|33z%9K(X>sSc@^{Z+ePNsf!9a&U{U}-;d8EXbTHDmpMux^0r7U_$BzPMFuUt!;!y(CHW58eph8mN!F9$8yL&Jb-?X>Fvchfnp_8GC zH7I&kcL@sJ#o$c_6jVcCpc2rwrvq?A>(y*FJKUS)#op&F9&z`g+4`*)#Ks0&gu&BI zb|j_d6#w- zmdFr$Xbpz;p4px=tvfPR7W!0-F;HBRL}bgJoZFA5AL11_yLXJ`KOtp)8b;*5GM2xD z7hj5^G9ze>TT`c!MxZYAP+1=cc~&wzMRcLqMzooU9<-$g3Gd5lSy4%%gi&vuLK3E- z;q|#pDTeWUDy;Vv1{E2uD+(yA45jD~!Z%T%?I>I!AlaN`}sRvL$-V`*02c{^Yl{5hB(5-nk<Pj5*QewE+g^G9BHq8|z&@7mZw9@<0zw;}Klj#_ z->=d84zHh#7RwEfpZnS%e;-Zq72VzI=Z>XPiXD{028soC@+a(N6HGyC7t3YAmSAkB zQ6Z!xX_+XV8^9-nUB@%yM{{!-UGv9u^capF!v^(7kKu>#7&?q#mb1uA-KHPWjq_`4 z+YS0+WB@XZuKi8?8NsDmiGu&LBmb{yKyjaS5F$#+h{y=LXT{of#?1IokSqrSx&3gm zQI!P~{Hd*gLT#-JzqwtD^&ne5tVGhGsz%h^1jo|^kYVTeDXDwJo@p8MO~qeZ`!y|e#~rvl&|#jj25i; zu>%*#yJ|e-qyH=mHoHJ!(HH!2)_B;_F>SyGJOFkD9swpyq<2``oHpdhkj%2~4j~LR zDVQoay9Xd#6ceUWC9t(t;$gSHdkyLct2oCF{=VkFeht+9Nya}#_2Hv=^(M+S7Ts&6 zR-)6Q)?|>shPAn|Xj5q5m@p2U;4Rj*D-6JEjOkk41rA7qKAIxaljd#x^_VB{`pS!1+ z-J_JU8(hdP$Y|bI#O_nofgYiNT@bxOiq}0x>AIl8bs=Tz`Z1%sOV$NANQRfIW7$0~ zM3%BFj5Fx{g!0pQb7#h+6`W5+HqL@clm^3^1YGKf%A!WauC!Tt>SJ#BbalxcK zM(YFU!aS3T(HJ~p7SP_F`3eTEPOtx~?1C!`OATKXOs=kmgrVLv->HH8q~#F$b`{#% zEo&5cSn;S4{RokFAgXEHC>8^{-{%;rYLyO%J02zkD-##ktc~*(Voar1=`}qQ)=ufR zvU>%aa$fZ@P%D7~7tU*-mYgLIbS*1}^q5hlie+@T3WaZd8Y-0EwcMPNuH)x$C-Yn% z8TA-@&S_uU=Rgf8(h8mAfJU^OO8L+k|IpM^-r0xWRh+cVo>mt=_SnvBQ*DN`c6DO3 zoPDZV<7hc+^!&51oRyLl-K%Y5-`j@|u;o{zL40PjivTWSNATkk6Y> zx7d*Ji4SN*Zg$D(QCCmA-cq~{Yeq&A?`KC&67euov!SQf3PE0rcKmHh1WJsvLOQXeM;jkS)7U&V8L zCYd^RPot$|(hCA+u@%vi;??neyRXM_-A1@J_jO}BAs)GwU)}73E~!3bTl6zM2Qx4B zM~JDk*Z18rrf?6q*Jjr6j_>RT4<9NCLlt{8OtGC1#eyT1ezPC_Mt;-ztw-TEy-mkd(THV= zbtcZY7`ezr^%m?10J+iHuYm-J3Gd08FC`WTPDDf#+v?epWy)!LXJ0nU=saP6xxKx9 z5j3jr?ZOh~RRZsNtVW#!1K<5&tVRowB;w(QVdP_2<(_jJt5IzhJ(4=71>3|QfA{`< z=P(NU)gH^)p~hO7eu4{FndC8AFsuPpAG^4<`jJb$it>WXo}PJ@QYNko@jk3h)suJT z>35ha6NxmX6UlN~P^=YJU?dS$B=T?-Bgx6twLB*0R~I+LW?UYVSfu(E&4f#qXx4 z)27xKi{G~a~8 zhB3adaH~g>TwNSQ{f+Dg2WdI2-Gp^#B5wL{_v~fz6#8`D(9w`Eg??*1R;jPl_`VL2 zA&MjkRK~Zu^~ovZWCx`@sL>F`oeZUUr~*$yl=yndqaGhLEXQRYsLPW9`aB8L=}CZI zPr`M3va>ybKm`_G28O*hoN*gQgEm(4sj-qn*ayh2HUcuTr(YO*+OjbOjcephLq8Fc zMn<}nLwwDvAoeM1UY*dSsEuY(--DxBbU*ryX3@@XJ(3&7`i`$@L4EbAcPTU(IPDwk ztl6V?n`u{Ivdi^7rdZ)$&`J=R=PZkDy&)y;SVq3RdFAC}K?H|7RN+BiNAK{3ca5S; zhRfnDEZ^=ko)F`=!eQ7<6W>sJs*scLS@PG0G|%XMeOrX5j~>0lS8iTy&GWZ6ufVXx z3{G&*L~DY89Pm*iut018Mf&qht;we60k349R21FSOa^uK!=o+ZlxO7HzVrB2Ww61X zi!5W2`U?bi9>%`ea>V;9TU*ROqc^l@NZpD+m~PXn**ub7vx8O>xb6Ejnon7uc?4}H zkZxg`%;V@Wfm%$M4ii#$*r&e)3DaG4m!`|QspZn?h=Iwv~&oStBGy4yk#pKOf{Ff)$VTfDyF=x!TkAPwxy=cRG zijQlZ$ki25@desLrD;1SkK~c8YYk|Q{Zu~t^H(P))8FnDV@H+#aO=PaUj zYeZDhL@cqqL$spVk`bCDRf+ocD1y$Whea2B%GDvD?rT_31FUnX%*(2AKhbNjYo+>& zSn1}xH)lJK33g$aVr`W(jC?S;T!3MsmCu!s3gQq&L9VV%aEhOs(yWqiZhXRHmYFav z`>+6i4niLg(8(2jJi=0^h;6 z3unM1kQH6dFdn8Gsb#yy2(F&_+K|zl!xUX2DgG|B42C+=-2(=1))Jvb4j9zrD^Eh( z(pX!@c;>8WyLw=7U>Mma$^$F+64T z|2o$go-&4~Y+Yk`ig&0Zd)mCl@D%s@%y?5+cuEiUZBtFc!p4ryh9*_ODM95g7H}4Z z2PcQGRWYh}qk1>;&r!W2qk6Y}jq06us3UvYyhioTy*@ME6!Zz6Nfwm_o89sJJFv?I zD|;7QLbfn6s)o`B`@2`C3#0uVTMrD5-sZ10>>cp31y90Mx&R(LhCh$t&->AD41ezY z)+2>Kqub%lkUBKq8fO&xE`Z#X|8{`Jc8L0A5FL8M7DGx_oSnZSx8}XhdKG%zce}b) z3|gr$MYoGrjOasTL?0SVuMvIduyKDqs2W`!*B#L`V!q^xCT}A~v(WO@7j5sf6tQ-n zYw2-MN$H(SS+{ks4;Ha??toRgtsO~nQHVTYN|Lkdt0}o%aoIAC&Kf&43=_G!zW>4k zE|K}#Mr|DHzio{by~Z&f4e?!C+pJNT4qR~CEDfPYzBhd1Cs9D!olBg+8X;RUMJPTd zHBVPO^)xEUJFH9cZmdH+9-+n~)SmPkk5HZ88tYJhs&%LjRj1GU{ZGHd-p<`0i_?AF z;&g{BtkzVdy7i_F1q&3ZhI+W4J~h^8^Nn_lk{I2Pzc4rC7$xCeW0XW!XU8askv)x3 z5^#NHylJe_=3JwN?bBs*qlN94Wnmj@v~~4+tkKr;*jS?tug|7No9ND~*^;qf?$crw zoz5v!1&<_>)!PA{DYop$upY~>jb+%zGHhcRwy_M`5PFSe*qm!D!`3;B&%O-X4ZZ^^ zgnmhnxL}il(HKa=GpT46`RnI=%GJq+7tv{MJDTSSRI#x~7|er#rDBgtdwC6oYdy7k$?Vz_~)qJx!0)Pb#-=B z??(1Cs&{aGX1r;vxHDGV!D|d=^Iu~y+b=7a?Q^cUvukCHOa>Rkcr)|X!FV&A`2~*KY!K6s;rgPzo zB2mO$GOIt1ZV%_WeP=Voaufcndye{G(a06n`f3Z(=vGnx6~M=Q6$+X$9{@O~7mwLV zW;qGXv=*XBr{rx0H)l8Bb)WXmj>*~C+1W99{oh(4cr)|T-lcJ8lXc+C_bb3P0(JDw z?AQ}p@?!eMzk4D}{IF@=+h#(}y6>Ewb>Dpb-<`MK>w#bm(r|SRaX%I`V?w>qF<&byb0in;`i{C91N0X{U7U zR;DuHGgV!ys!e^oaLQBRx3A7#w>IjQAA+8MWnx9|F>a1UFTT0}`pnst9gW`LDVO@3 z(au5~{bYk=ht(Na^kTmXSy9BK{A>4tx2ZSSr}eBsUB~Zd_>>P0P5+AlD|A75D0oXN$q8qo_NimRD^R-z(QNa}}IGvKSd&(2A zKbqF;E&JVtD6{T~Sh0fDTR|`WDSePrn$mxYY#NF5V(Wy&%v)U#I~uu=GM*QV(?Rc#7s(@n#Pn-Qk^go3p8TeO-Y zy7R#>!)d>9G0@Hi+G~%EHXsXbfL$*n6D}mBLX*IA8wb{NZuatn!=H*flEWZEUs{7a zs#x6z<4W~9MhI+Bq4dIH6slO+Ope&rima8hZhDtV4~XmsK3f+XaqXQwr^?u`oC6k= z$^Ms$f-l%Iw5sZ@Q5Er(oj)iNc6Re7OjG@SMs7sH$l1-CV?r}RH8aep;`a=__j+kN z>T6FqIWNw&NmLB#Se4eesX|L{D&wd{cMfILqDL+Iz*_XK79S|Cc`njii^5jsxRs`t zkp)d8J8qE`kg!O~EHVwUm=sLrB5MU(sNfY0!o4v#Y-Z23aLh+@Xyw9oRCYyRnyPNH-98jC|$H zg>@m>$`?qFiVjn9!y=KUEQ`HW*$D*L(YX{!shC!TR&q+dU+KoQ@(6+)ekGb?A6tip zA<;ySjVe#!jl2+gdy|yjF-bB(mL)A{rkI}3xhSz)9c~3&KYj2YEyk9 z#4=&zJP~E=`i0u%qgA+2;8zxoIffj4^Y-mGuP%Q&yS{#Pb$)hx_4dtA=Qo!Zmv3&b z&R(@U%-U)#H0c%To=um~O|UvTdM(SAJYz^iRVq>Fo}ajBJK1ug#01)y!?q16x~c2tfc7Be{sl zJ*V*YyQ!X-DV|N`J3NR%kqX`KcHdp^aG@6w@qHILE!Y6*ebbucX&EaBRy=2sWp z>Rn&ax5un(G8%}3D6HsmhBz4_j0HhB6MW6Fiaxkh6&$YcH7|upj|{%u?&=i zUHkt}k%h&9>3-{nD{E>)F@wFl7A46T1eYX==Ser`)D6L?t4AB!XxBp6e@pmvakHUY z@98cdPEB8lOcbv)d+O8D|586mqLmQR>_gLWdMw+q5}t_R33Un zf~sGhGZIuY`f!INw>wU6YK<$rLyv2h(=GjLfEpgWY}E)#=o{*oWj1D+d1yFhnZY%J z5`H95!ny9VmNa43hOh18u)&M8RSa2i<8`;RilH{%(IUIiOT@F~R$R4dNOo=N5H6on zDIY{J1QpJ{;zxz+v9h_;u?@Bfu@U&g_uweS_M_h@#X7(BNK&kntQZzL_0GO>Fn(hj z9Qlp8Mt(E$o3;^s_WY)|=@Dl6ZBfpzd7;o(78u~CI#I%2RM2P9R#uDOVjb6OQ65{psiInLb zONfsp#7Fjo*H}W_T%QPU8WCbf_0H}4$UlEU{BuD6%i&^3;{xFBpDK8yJX(By6Yr* zj3xu8=Ug-T9MaEHmWpCMC8s21G=myb@Hyit@1|OtH~Gs}Z!%_1Kq~$pzUZJ?dzyZ| z(?SzI|GND~OWJ4sl+Cql8E0L8B9Tchl_n7c>se0N$@oO!*7sQmO=+xqL$gZ$xe<_% z0iUo%H}7X@uGWnp6D|prvP_Xt7iFfETOL=MMwFCNvBDF&Vm4%j!%z=h5?U&eQe{x3 zXcpJ;HzGdj)-le{8k(_waSr)aM2S0QioH=qPVQ-OQj}R$=B26on4LkRNR*j6*}3#A zRfo4|Bv`Mwq>OBVs1s8F?zF`kf?`~S!162_7F!wH3}*Z4M#x* z*O=$tb&dRHOmt5%%RTx3||Xg48}I>knUryq*_+E1-{zXo9_A2vQM- z*>P^Oy{aVv$wfB*(zJ;`{_g$zPMQ|>tGjoqC!YXXySZeKl7pwmVaw>7D}c5HX@Gb)$%}9v;u)sFe%Eyi`hspeG@F}Xg3uP z*&EyHJHlmlq_z!@@QofG*ysV_Ss<}U7;Ix8BMgCTbVnRK16LW8f{;zinI2~CdJR}v z1j>QOguUl_Kbb^6>n1}o8{~bO_fYL1=G3tq$Hy(lfd}njEbc8>2VP81n~u_H^RVAJ zpkO0FZOGosb`3#US3~Ro_7%Tg?tyXAS8Kq(9ypNih3VJPktA0a2T{()yo50?VGR5o z^AgCIm#}?}c?sU3j_hgk8uJp|>oem`VQHtJfDH`%x5^>D=2Z~;lr^tTXn?mzqpk13 z(P+CL{YIm0=eHioNo1o0uW3R3CZ}Q5FG={cZ?LmwkKSz-+`)g;8_I_@BtpMDHRi}R zRq&OYS6lP^?aeDdQ)CK_i9HhyVZ~Yw_^1((-$BAC(vQS%O*TCbcqQ|sqUf&H%(1xb zSlo6jZaWsY9gEuzq1Ra4*15*wwvXcpJ9u$hk%Zx}-iV})XP3c{j3s#Iv34BmPZo zpYm^Q`&9P^w@-9$bNfW!X1CAvZTZdid>enO&46e9Ou#|^%GviHT#qv?TbNfE>&tDM#9MwDb8r8e5&W`Hc$eu>^4zACPH;om` z$BN~6jiJf@YYa{PWrZex&K1l3g}pmmQ9D-l1|Rb)p7Ivn+n`QcLGQYlH#8%gHM(h} zcX8{IgMvOp(lk$4qiQP8+(4Ln+>(;Kn7;UOdI~{D8GC3eNxXZNZs@vRC>Y(lD=iXu z+o1)l8qe*#a8bRP!O{A9bIP2&WCsN8(!K0h1V0~Y$_t)<8 zw%9u+9VBShfq_agXA2aYOD5a(t*{F2Pa5B{)!XBBMpGVG z-W|DZ>XycrIGMB0_NzTJwzWJL9Z7bcJhsmWO`^LIu>o!XY)$9* zeD`Oe0Pxl%0Jd7=nV860u|^_`xK?PEGb)fL4>zIf?&1MOMV&Di2<{SjVZ%A~4goFea(z)82pTAKzv!WH(`0t(};4M&Pc`d?Kq*j_aIj~$b< z%g}EPl1}1~6j66U-zc0ab;Jo z-P3w_u&GV*tQk@Fgg}m}1S`TIvDP(gc;HKcjC#KQ)riCa`C3O!IxCMkS|-~TBr?+- z&%C`uni>^g{?Z$Q%H{|QSz5$gTx43!Q7=&~;_b#RcUOzZ_DMY;f|*QcMyao9 z^e`LdLIu0sbNgQ54dR+ZxwK^G*Uv64P)3v@LY}w;WhV$rbWgHO|F>W%5i;Sz2S2PL zE`Gw3f1OYM-?zWmAI>I!d;80u`a&-T7zFx3t47m)>BECJ*#$fb`3y_$CWtID*j)B- zPzhSOA|7VJ5U%qo7o!6lkfnMzZA<|f%YAN7D(2tsr>Lo@Cw2Yxq^V(QjkB(S8jaJd z3^!QSURlzP_7!t0l$W?GUwy}Y5T0~#>I&)w*;}-pW=m~aW}WPs<94K#Ui?O+6*?l* z#T1=dY8~NQp1J5varltrhVHa>jAqxghOP94osY(fm;PT8Z&k2(XCwH;+ z;@;$hT{m9nC>K{g^d76lZFS6;8~SU-!E$w;%&PmZ`5GfFl`u=RYO>#YapmMctXy1{ z-4KRZWQyRw*KGSP!lqgayQ^U#ah6qM?`45tBFi$}OhmPQ( zKkUL3Zz~{Jw>$wjUFkzo7R-7;c4B**&{OiPEQi#ey`6gH8E3M{`sV`1z2Su@Rjxy@+$q5`lw)R0Hf1$sc+wUy_7qIcK`83^)z=-KLdvYD zsEbkS=pD+ab&Oia{u4d%{y?j(g9PTM%KT@Ib@mKh%xgw|uX2EDh2&3=T2fp1T3`JM z29ufN?WwC}EJ_edFHuosy_cD=%}-VhjcjdveST}gFEp?)WKN5pH7 zX^V{W%M1U-+pI@FbP_kZ;}+8LsD+Y z!@%IyD>F+!(9zkiM@K)MP5$=FpN}`kUzb1M*M3fGKQC)PuWCQb`tRlO*T=hO@gj@Z zGVXedSC-W)c(HHD5t&c-mtd^fF_(d;C%@Q(^N3y8CHGu@0uOw*$+e7^y?)gRT8178 z&fY_F#sH=e#09Uxn44<+m^o$)EgnOQ$JjkIh8EYpJBAjQZ452;4s~Qt?KXxM*V{+N zo5svBqk32G`^Z0kNBnbC@9J$-@A^7Bs&^xM8r3^!9~o~NGsldXV@w+}$N1ZrIcCfp zv)w+-%rOXXXEYAi^n)Z~cb~cgxJ7@8+4#O~9DCb2-aADWU$qO4s+VSouPR40MHj4& z*mE#FJGTh3kR*IHqs$M9=kp2%SOE$zfIvE(lz^-GAC^CZW8i z2yAR^NKe3MRA^pz^kge4$#ldxK7Y7+>8l0norWQ-C8msK>?G~8bY@67@hBy`DkDA& zZM+f^4Q-=_NrtA=O?68?5IqHm4%R!;AdssNPW-z~q+t`J0Z^8*#k4-?H#sqhL2#k? z3M-nx*~Mup>;3H@ZnDkI$vu(^vlNT8UNg{fW-a`x~yxBuj`xBB^8GfE?ry40pe&6c(;V#Zu^W3IU|*W8$EuJOQ_Yp(Ic zm}_p#H8||#d zidmDMk|c$qYuEjbL-?D-UT>Hu_X!V+Z4&Nu8Xe0 ztTgwzM6$?pE|f8?*7h%Dh>O%5=G?kEI?Q7%PF?{6c*$u`V+c+ zlM`msc;1kA>TG&8eRA7Lc>Cce)3fQ>t^PyRCm2NQcaHEEw_Wa^KHJ3vwWxMJ)7Af& z2chqQua^Dpm3KYj(5J67oEaIF%6c;-Y0FJ>o^I%h@FFN3>(4q)RTG6I;VgxNdylC? z1yi)^HtPI8_wnq)yZMUko-xQ-}KkS}dH1l2oV=)A#Mcfp&?@bb6Dd~b4xD(E@ zGA)TS$Aj?jAtwVks7J}WV!s)6oq1_@ZJC!A@LP1qrWtl2y)g*chMcmfS^e$Ww)JH7 zbPVqCE`~eaN$hPh;!bi?J=tI)_A1ii!M)cJw{DMq9r5I32xMchu5r!~F=>7qVq#l1 zcu#X*>V?Nhx)4sc5C%L)#Nh1|eHR8;#(}YeQ0#}q(nZ34!=UL86UAiOBLJo&=PeE_ zJvjO;B4T4mY)KqaI*gO{qokuS(z_s}LoscS5zY3PNB7||6gxyvjEvx8U<5nGOhmn7 zv@A^%E_lwi&bDpawr$(CZQHhO+qP}n-m~BD-kIqi)fpLCQQf(+Vy%2K;)ze}R$gCA zA9Y_588BLMT!Njq)3@nT-$;nj-%F&fBCoR|uRh?)3sOCH`UG$4`}N;*?`foCb?xXU zR5@*d{my*pj2r>4(oM6OOfvoKeoV8MQkofFflSiLpij)woDU$j8QE$C2&f$!QP)73 zMh8pr-NWnP>QF`3qzQ{4hJI00>iGxt-)i38U|*E1f|CqbUo9b%ZXvy>J%bjd#KPN0 zXi}w5A?GtmF5Kgs$zD~hQgbe<_``5%GfXRa4vrm%krCw=vUc{-3}&+k0p!fzPVUPp zPz@Fl&eQ|X-dVh*yChvUeZ%58_(Tv6iG}+BG@)NpIL_8%@Q_3@2yy*CQkFq|NG6JQ zcqJH};D~Nspxo{eSe@l>@>`D)BbqfudSt+=zQ;uq4>O=Pc+bUzu#9h@ML;9 zKM$$seE!YidK{puiiL>a$;^V|OW1tKRCMKj`TUw}yzjvg`)EWo1=x{6QKX7c!waib z-8So2VM2Kh+)!`?*70KI;vl;DbF=a^NeFjVwxyX!RSb?2DDjolqQ!BOFlKl! zMs${KUM8|l)-TC4*3sjCtHtvsDA8MTs)mQ8qXh?O>0NyU(b~bI1{cI+GYbmMphC1v z&zXUQPoQ6qYgx|rU74#U4g~9xqn#s_<7rJtY0bJ&JkaC0EICM`{KR{e=Oo=rk`8!^ z{a5ml?A&`a){$!@oU)ZPsRjHz)&4aiEyrRZMkSd36R$wtYv*nxWleAz@2(=;l@a;(@Cve~yo`9%*D$wEV&jgb{dMWjLKR8u^Xs${bhvisP{JkxdH~q^ zEY-E;B=W$SCQin9U5EpG{`PYaa1#ss#y=9T{&PM_jYSjtHn=2I>DB5PeZ^U-xnCgQ z`91y+$^|F*R_u(sz)IIZ#GXy$s7e_Mtdfq(r(E{w@l0dVL|T%yqREPJf=K?E`-Bt5o_WWBhWgLh&lNty~&S5oudksq7Y=p*?UN1*?Lta zoc92yX~_s&b#@Z6C#6>*Es4@2YwpVvnuMBoXxuA4Ma05`i5$!?B3~oq(=FwGo9Wvi zh7D!qPdd(Z@~mpz=a`GiJ}}l9F)DGdNZsX`xES}fO--?W25A(37IkPY5);*ZU_n0K zRJP*YwZzTGC`Od$=r%23mLNu^UFJRNVamz6V!_Aj+umn$@nPkE`n>GeL zTFyEi-KbfvKZC3ppa+$>W9%sv#}YH(4OKGr)G+EcaVzy8g(D7r6*9A?(y( zvrJMp++cv?Fd}DqYI%}7+paEdHkJKUg?qDBe#{ZFAGOab-bd!u*84J4_{%CbXfOlf zd7kALHIOixDV~tjb+F))Po0B_q#RGIWnfWGlP~l@_BWo2dXObd-5tZeCmg!Nb;Izpy)NH5)g4g%i%*|mWqsU8cFZN9&o{Pk)A@k`P0B~YMA?yn;l;I1o#4xg~y4g~|S{gpB z$(U1)-?(u@v+arB1Y&YKoY|o~XC&tyo7HWq|rj*mPOQpWW6HDllh%bE$kHd|@D4%F8gd~{k&^wVzHLPaa7B&)S}Ne`uq+rR5V>iX!}sNQR*Il2e%1Md10WB;Gkuc zDzBK?=@RId^XUx+3f=Yv3U1nR)$n*({sTO;j(!>`QpK>;Y{vYNCxOw<2R=ie4xbQM zn5?Y{QjaSPUB{0nOudWjJhz|JZkzJu+%a63SSrq8eNTPDyfLSp3VAj4vP9P(K);9L zye4qrrYh-Q!>-uq5=#V9U@Xv;`a)#B07FgBO_5OlGVe5Sw$kSvvB>!}bbE9XB-U|% zv~{DJIIvOm(rme-Wc|6Fh5%gn)nImnIp0?V*eLc!rU84_R17tiL{C}ODZ)XP0QFCb z`X+BplH&)XIF*HJn*T_K{RNZsE~y! zF?E~kNJi!+``%E^CB}8WvO|RPZF`I0w-VSjoX@@cOCnEDmj`+5=r$L!sL4()GTo@G z`3Fc>Xc{7%OSzu4`TL!RR6b(<`|=ZhvORkvAO@Fs7# z=Yq3jBF-+YaOgkr6r5aC>0%poNpY`*{;(&hhvF5ME4uj04CD;+9`3^3;ZV_z9lGk* z!w)|?B_#OZ!d)1NT~Na=Y(}rX!q4A24uniTx`lLMA0bgB_+dUHu^>hFF?%S(#oV-c zKBo2KZZR?!IJjui?>6eY&${c;z<>Oy86EIJ6aRuKV9FEsHf}`#ry^g~LLoud$^=b- zraqGj0ZDG;CGuA4a)!~6!s4&VS?_+)t%5zcnRfjo4qz ziKadKbyHvo*I&P(ZAPuA&k~fiJI&0M`%vLJ&s4EmT_PI7K)&Qh?>2&Pdpolr)x4@{ zX??yi|Lqlz4CD}?+idm)hGA@`AX}m&>8J4le8?}5-bWOn2lP8T1;T1mg{$Qw`C#9l z0zyYhvA5GzstP6-}G$m+{i=Dv(dhoo%)=Ap_@jpObD7&6AFv z%iG=zs5%P+ZH)^?0`23OiOl!&QD5CQA|#bP5DoH#m4x$;<+(wIeQcu;bht1EJi47iS(+kJ! z(GEH$D?hc!=n+^OCSHnHOl0hYv^&9}{Y4v`J==eZBCB61P_udce*-qG0yU#!C1wb| zF8+b<5-=?IZjE=!j#RT|AV7U8@5FsEfX0k!q(T$yI>aY}<381mg(jK~Kw3cIPBZIw ze?43Eq5>Qzjwd6)ty<8GbQwlLu^3!vvHoT=0yG2O;C#G_+mfJijQrDr(-vIUz8V0V zL4wUCzi;)j=+&xB#s?WnhWVb@(fp6c{57$$v5K=oc+)gNxBsoZpA!Ow63ocw%NX-VO{ z;+bjA?~kM;lenk`NEiVJV}ptXEGmqR$n&v4lM;q>!-p=))4Q_l!i@^v$)IQ2)7K|1 zP&PEXkVs=>o-P$|d$>nustoKatnun#0>R#vYs@XtnK4&_q|(*0io`&3H$IQ^3k1Ch zRH0ahT)%k4PMqb;%7uhM_{6&k!nm;n#8 zTZr^lUiA}iLg^yostgP-299kaEw6r^gitej%E|?vk+64d6K3Sjq_1$BqS09GmtE)X zXj1Q(RuaXx`d%b0zCnTeyY>z>I&9y&U#j3doL$NKaA6wzuq;2O0LXhJv7%Dhna3IX z0kn}3Z)o?F4eak`y$_5k)M zDFo0tNPi}0Su}C)q5bW3i^{VV&cdg6rrB8bu-`ET-o>f(XZ1mSmlYTX_L0N=YTx_! z8L#rsK%RMgxKs7+9{ezJ_i4Eknz9R@R;3*EemTFmv{W>)3qcd{)Q-#nHQlr|$`cXB zpp*W_CQ};ntz^M+a6BedC7EWiK!(Ljml+7pKSy>tq$oaY+5vHXUPQ-!wvip%8=&f$ zQs}_b)zRigR+rl6i7`Ef;VB}Ar8$K~s*~c^Vzjv3h(bdd3Mp$^7N0^@ff@I5?>Z#I zILcohLFqWzgYb~`0_bhI$88w%u)q@=k1MLc0h-c|<4T`1vw$bu@Udjnq9o2-j z?YMoSb!ubqT)x=Dvx8`7B*W;#doOAl?lkDqI^c{kCLX<2^JO{-VKrL29M`RDje=Fh z$6AiTJAU*F7rK~(K9B2`%dM@;yQ@C}TU^-vIKebzOK2Ma{TMldeoUkpYf2x@^OrC~ zZxJyJYaeCOD^P$*#Fn5Msb_=byE*1x9<>m4$SbGX7tJU3N=}EJLhs#Dn1g7aEn-er z((cR)H`nBLA)I9meuFb=;srBcZ7*k69fK5XQXF+4l{&+^mZ)B!eqU>jdfur8cU;SD z=Ha~efm>gfPv=>BDw$5)<8Ejhtb6fK8 z1Px*mP82d_HI0qo&$P)F7pA^ci=yr0Glh#=|KkSd6S0L*-n6ZbG(|T4rWAGGth`>~ zYl>1R5ocy7d<{|q6{_0(s#55D%C0y@Yqh2Xf*;&A*u!kyL8jLqmu-NycKd0iCfOR+ z(&fv}=gHe(VbiObRA4#GV$aBy%5mHYYQgcQ=gP?5I_FXJ^5<*xJ^}Nw+c5e(k0?yd zMP8aVX@AFgz*tFo>jDNI-61ekN?OXC7l6IxdomtSj1nv|x~SK`Sru)%B4Z zej5sXd8Qx^q-o&iv3TCfZcW;L_d2f4ATr|wm4Enrdgw;gt%;sJJ#Q=FDBf5iq_L|N zZ(1rj3KPq416YQbgmq~%=Vd?BD6OYlF%e2cc!l&wxf>Aj`Yvi4v> zg6+hnO{VzPb&1YsI=xO=*xW64F?2AM|KWYzjz2CR1^**`8bCh#0%p!?0I~Cs!Eq|6TvyjeT z0HkaP*SlF6stTKE!F__U^f8(78r#2`V88C-%;@?73ebn5$U_*UYdhR*6!+G+cS4bQ zk@e%l=n*XS=pz?h7OUVGO=uKRY4fwPQwx2=ZYPx@UZ|oM#X!Z~SU&?h#+&*Gd-OnM z=H`E$q6vTD)f~JRz){b}YL*$>EzvCqigLMI$e73iZA4ari}?NL+nII?zLu*IP@kng zYwlqLubYkO5WjkRg{t$)cC|gs1uNpknWbd`)`_KtOQR>_C3WlSsiUG|8TV%u<=%Rj zg{&z_rg^N1q$d-!kFY_ zWpTY-!kk*d>_9j3lT&`%Itv3y+0Qu0@4an4yf7?`|9BH#Ex^kX+P?_k<45JKY4dMJ~7RV!_+A6$m<{x;?f9~xNDk+AKO=Rs<-f< zA>XQEM3EX_Ln0XNHFpM%E@l<8pQG+oucikz;BC5*CEYIAehzsa^IH%AS zjG9a6RkN~s8|hNYoDL>k1AQv%O!wIQ91%6e=5kU1O{Z(ISlWyQm8*S` z#O^J{E{-e4>CL%bA!OWd9>V&9CThn@Y%SVG8K0IYK0Q$^c3$e)v-!#7VXNYprllzJ zWx{{g@!+GnzdUwN&)>1wq4-IR?CkdZd*hf)qs3@L0^?t#SRR_ z;u$hpW`QEsriMKzclkS|5aq48?2`luA%;P|)7RWF-Pz0x!<(H&F90+H&e7BPdiQWK zT+@!4sgmJE>fVAXQPtsA{w(%e#&k+ePtf20t>#jjg zm=k~p?bu4@YZw}v?uVpS-u9gi1`OG#`=~>71&?K5|J`f~;Gn;1?7~^+w)7Yl$idrZ z{+Jn61lZo5J1$leSfdPFVD13nA36;&6vQbtmTp)QPfe^!C{Zq&55XsGCrvWzWDKuR zH7~QJoe+ZMezV;5H_*=nxKXs$cs9MLG~E<5wcFj!OTo&a3-xM^r6qryT7}9ln|A!z zR=Kp2THVO}6(VknIT@t^NohW*Ha3>1yL?>j@R#~q!Le_ap%Z_NfpV;X#y8uHZ4LpW z1<1*&@auNjYK;uU>%u6vn>l^yZ+E6=dK{6qg7)#b*~K3viF$r={P$yXwMkx>%InJ} zvkW6o=l1^hFRc>U7<+#;cDlvS zg(9W|CC7oqTMLox<9VBtxtsDc1e zUknVbT1^~j2Y;eEsI4eKW88wU^QvZ9pVi-j#m$qavhk!WWl6y5V)0WkPBk|vSrLhl z-Nj^X$3HeW9)IG$8Q6i`vG%^$3Hl2Viqzc5xmexPj zVtDWmweE!+91kHFcAOU+^r!p%gkV`2myGd6hwa$y*0!PFp)TQ!T>XpnVNL9!`B{>b z7X?mucoJ}J#eV=Y?YO8ArahZT#|KOx8YGFAy&0A0v!u{+3_05}79ufknCWvEbmvo; zoJ1Jevj?Wye*@T3dn4SEdZtk89kTiDdx*p!eGiE<#fWIH9)Uhs2oA{DZmvd^O|mgq zvEpv^*#g~7auvmJ?!T^0{cdYvsBtR~{+r3l;q{AUg?L%=}PN?v@^HvOt>s0-AydDy6447;7U0%LwwN zsU~Cjwwn&^L4OmkAW5!SY&Rs-6p9rOz$8D}6qx@-5KHi6|HMAO3yT&aYf^l*dGMo1 z5Xr!1#h1Zdu%69`gqmxq!R7OAkOn-Nb5uv$vf8gVhEy(j$c!IsZ%brB9z)I&lc!XO+TPBsX(5wNMM5PydY6_J zti(6|lNM!2H%80dUT>#h>^_Ia?k^?2lou$#y!TeO1LqU1DtG@+W(4-&sAZ2pyFAxI z40|bLm30jQhm{oje)F;xVpU-9yzqJ@VwJ3zsU;`v-m7>bbU@oJGmC6 zm#49Bkzu{h0(tG}N+2@odVRK0Gr)4h?5yca_!inauy8PlI}>5s~Pm68%ln3)G;(ynN+JjHYYK&QmEhAg<)9tGMW@n0U1O3XG>;h zW(7H0{FC^8Gm&G!Q21aZR#G!cc;q%VUN_l_R=asE&Fykeqr9QYg8Vlkw1u@81XRv) zfmC}Nj3ftE5!{wmEqX@5c)6(S!O57FLQ%y}`xu=r{Jbu(7CrLJj;9K*fid-zS>q@} zrVveaVU57jfjvL`FB5qXvZHrl^o|v2T&@ QK{=v&$9FtyouUa#op_G+n z?KLcJWhs64%y$A}NqURSMQnMQinD{X$Bn7D0u+{W)yhCXZNj!@5IG|CllCl1=Qd+v;{MFpAyxyI1fUh+kiv%-B58ibT39P)**j!9Hp^xNX(s8r#02( z>58pY=fcN^YspLFoixw(nZ_6P&Er9qcjtmjqt6S+*XhddT-Qgj$CgB~%l;BHV2w8a z-~s~ntjrE^>h@ct*J(Do$t?zs$-QG;3OKKC>6x}bqN*ajFlCC)jVPoIPGH3@ zJswd$F{5g9E^{5xV+yWK?RLt`GEkzSA|O-Q=|(-EujN3?|GramY9!jlPs7s3Dx}v} zi+bmWI&>Acn>fFHE6I#*e>}k-s$d={{UQJ$oDE2>4u3*=QYXZcaAi|2fXzsAA(WmB ziiz}cUtfTO&^}t(vjc(KrPw*T_-st5Z42c5WzH-=WDD0W^U8cwiUoh@kvCxgRsv;x zY&|Ggy;Ih){G5+a&P4x$4}`YyLdh0I02Mv<-hEfF5#I1cbr=NxU*RNnD~(fxRDRc% zvu0r)2t6a=({pbh3xd;2`lK@}0F2mH*VcZ5+f7A9)8jUv7&j~0tMkU#cc60}U*6Ws zldT3Kx@8Q1fTg@?s}{t+7O6ZeVQfj=?ycZL#+!@XTqdmaXHM;QmW2XP_&kw>e}s}Z zl2O=FoYX<==)II6BuX*GoQ>^czb4r+zFx1LRTf#p|Jkm9c-Al?|TC29v=YHqL(UJ zP>I$XG#`*YSBk0uu1FP*5g~D9iKLRbYpM$tgFPc_X3A?TtryurS2>G}&jU}1dFUcD5oZo;7 z=6LvcU!yo7E7-N$qtEw^-C4LfGCWR}Tmgf*qj1p1X{0|hdx5aY;%Ov$mRxnzbm<%# z6MI3rm80<0&0PpvLAk~Bu7v&9u-V*tF*UL>V_47B63yZ<$xjs9b?|1V5~%g09N|UF z9>{ZRC(X+BmYvH%bC!O4$b_BS(02ZH4~q}uF`kh_hbN8;j~QRSm1uC4Jqv*nqDXjj zCfyRKVs(1e)*@~%VpgIed8+O7LaWsBDE#uDpJX{gBVEBJj(yadulUem*Rn<>AUstm z#K&nh&&9o2fNGxivCHEm2KLY^S`SD_u{Yj-C{S5lqKI}Ht2DVTQ zj(IZoza^E+d|xVE#j>B2(S0zhXLaLihP|0WkP%q)Yg;#530ja{KXF9$5g;uwYnPH3 z4Qp%5lg27+s{m$KoQM^HE_G&CvpXFU&uzuLoW3N@qli-t$)vQzB(&S zG~6*Bw$fRp784yPg?Qg=D{F`>BR{A1o`$yks|*4F)Z_q&t_X>(pH}`--8K7eDF8invR$#-1uexSFMlPB)csE@U8C zr>X6@XVz8eNr93ch&>A+2>kYSBY!=R>2Y_+SI*_?K$FbriFe(fJ|glBbOt!;c_YU&5PLvG{+rOFQ+g*VWa`Tr zr2>lW3y*`j_CPAf6@xbSP}eY9MVcKK;SHqONP$iw+CZUBf*6KC*gj=3(8(y;n6UEh zY8JbHqc>Fm@j5JuI}tNMMoQT#cgH7zGV6|0ul0Ntkc! zh}O#74h3N4h!(hUN{^}qQZS+Gi00ThXEbYGFp;|?oYL15HG#{$Y@nMdazKvkVzug^ z9LOsiSiKrmN}-@)8A4)BAlmS8jugHI7QaPRe9*!`3KhENn)w>#SBGI&Dzg~&`8Z^S z8fz4IM#pSi0Nn^B!%wPR@UJP7l^(7RUBeEQuO239A#}6PPVbLh9hyhqiwI(iUih{4 zZgKa4NBQRCBUTNYl$lX!+az7fy-X*OfqjLXU6WyDt2u4Co)Y!=Ip@G4DT`My&&c$y4LEW$7lH>P# zR3>t41i(s?+jyhL<-V>CetJYnU9U3mp`)yV_CHhPidMf8UH!$c>L$z}CdpQ~d&}B@ zDDziV1vhaNj+!Q2_@9%xkFotv`t>qMmj81>OYE%w`5#5ekJv1^|L=~QsU?kQza@B zSt-n$$#!!#0gHXtQ04&xkfp&pS*nVoNG7}CbXEhYOkNC&{Yda@cTgtcRt$vAsNd#B z>MnKrZ)1P`w{iQLx@%qWD-8p=d|6e7J?tSnqE@hZIxb=U&j6;(uYue;;^|SLk^dR2 zBbHwJKbjTc6h6$cIV0M3sNdD`zeV%?-x`XEp$@3}ZTzZz{(Cx;-TAL!hUGyti+#nb zgYvM0H-hO<+1~_=kH_rds@%86tPujR}o4thy8x@O<0!n;bv? zcF)gg=$V>wpEMUg_03KK)A)u!nq*QDN?A>1T|qmV7jFds&cq6~;ev~SOR!fy1Q1g? zg28aOYh($UkuI0yyY;uT?z(*?9WSn-h7)OL2ymaMgI0*VIvU3UTA6Z}fDfUyBYFxg z#v{TPagsH9M^33Giyz&eotbHK(-nO(9)(qYog-d)pEu6cEcb8+$@Me|xLSO#7X8aI zv0c>k6?$1_a2BVc1=`0a0yy%w=UXy%ZxvdwQMqiwv!?q?B`5Kg0+w%HJ=wNfLB?NG zIg~#`+yOF>>h%%*$chL^FU@cpN{!P!TtUwd2=NA6TzrZYqMOB5gWS>{m>7{^n;fe- zrqE5SFE#dc#)abY%l`S!DIHlF|C>uG%@g$V2=#V6^}r%C=a#-MZx%%OfZw1LS`2pb zD1!f}^#HUbmzxf6QtI?JySsl0$v3Y6lI~cMe)lqWnS&y6UrpMeiYyAlukz>3NpEV2rF}-5?O3R_X zC4L2;{pJ&>%8WycWNJYF7fJ$PS=$c#=yp)uAcEujx>I{AY(tU|T;rjFtd3k~^4~fx zN0K@71Y`y_@za7W_Q5y1GP+O|qoqIrfAt)`>0qrXTL?w#4^oFOHq{ptqJHg&x&v9R ziGb|_M-oXOGLERWh|^9+Q`=`c>Id?3=in4-dK0rUTCEm<Ba4&0v&+V6ZjoSJDbJe$6+4XO~6#ZDGzyw$3$Q0Rk)&SPptYCpP_q9o9o+v0R$13r-qA=ARGYLHhS+cv+-i@c5)>F} z*11L5;UCu`SNF&;9)7s>Yrr+M5ixF?#jwCFFlD=SG!_;&CrU*BD=FmyzGT=xLke2w zl=W_{Jz%!SKc3lL?$s(=x7%%M5Egb5z&ZKipr@m0Hi{-W7cV(#)P*6{Lw z$shRDq%TF}&8LTQ;*X zjr~)2-BM6MeBF&{;=y|L$bhWwFpMrn(*kL7W^j}(p%zosEviB39R*u&&0%WWGkY zSAk@F2DGt2N7k}-R|LnD0M4HEtc$bk;<1M7MRCo_|W7P#_>oq&R|U{-Q;ih z5UmQtY>R`c(38}e!Ig_rV^jTY_jbJhBo&MC7!Pe5hHCv=Y{b1yFcsqw{^|gpZZp?M z=g}dI)OvPj1DhId=AXNmjYF8;Z#TWUrammoY0>M)0eoOp0qdZ|MOF<~lKu6SQFSC^ zY2uOhkgkQBbB3QVj!x}UOlugmVHTF~^WT__f2~^$x4iqh1z*F7(|(Iqp&s5zIm9?dmDaxike&~f))?-8;R1F*HLJdIt!~c z$UdEX@u5so?Ib**B-o_9?_ADL%{p}YBi#kG;IaS6va#@qc*YKwr|} zz!%_+C{{z>#M9u@@y`iRF#gH~xpAYH>7D~&o$6GR^u-Ozf3^`%JJk8XgJ_)!8xK1K z8VBgfTQu2Wm22Q_R&OX&d@w5LcX=#NqNd7T*;(6Kgnc75Wubu&=1hW(2&l6aSB`V1?8pk3>}=oWwLPv(lW+Z~ii-KL0= za>V11t*t3MSH(pGn2+?&_}8Vc?uo9mNW56p6H-9&co(qr5=Da%3CKS z-M@uFb}@$!y^9&5`TI$lU9)!#AZjk_sv57TY+sOatu#UKI(obLEgKu-=(P&8&U3mW z?O@p0hjI3}(Z<*(Zk{KlGP);>ssI4l@W(ox@Uyk+oTjlQ6EM(^`o_wZS?qJ>T_EY%ipJQ@X}w z(fXR5?14iKW|56lv8w(j=~$mS+_H)W@m|icFtNHUGKy`;@_~3C78kHLdh(|GYuZ=0 z9lB^rPuVT+v1q%nu;ql>{(Y1ygILg)0ZhNn*7$>jU#Vov68h$c=j zI;Q8_K=B1VGwy%UCdJ=m^7#0)dHS&(nJE+`805#Q;te9J=K_Lp7qcM*VNjcvIxNnR z+6^E_tMkiuc?LHR={SDJ5u8s90LeqBjyfGt!vVSq=3M70>aK}B#cHzX9?QnMcc~3X*NpY z^l}dJZF>SYNZl7C>Jjw(Jn6GeEGZ1Wc-^`d0vH92wAU6s-OcwI|JGxUx|cR)*{RI6 zHDj}nju1iToA2C=K(BVZ>>zm0!obLxG~#>ehnR`Jvposz0Nw=x&sR1GLN{6MICvL_ zBi8;tzrm7JAVeWl+~G5+e_oNl+w&bS1~Bn!;HoLFA()D%&bt`;TAlbG1{PI6E9fFy zjvo#y%2e2A7Yf%5<(!y>fkR9&w~W|m3rOS@rrI|N$`Sm11u?4>L?9eBZN{aBsg~^PQZ4{pGDJB<_+^*=JvEXu8 z_6GZ&c6+bMb=jq&4NhkpH?Vj*MY_*TClFdzf?Y-q;;mA4-*?c}RZgf@v-jIU;U)>fmyz4M z*%QfYkPP)mIrkx!7zmM{`&~3)6(c_CG^8N+UI#9zbtn=WZ1mj5n8`6$3e9pnADYqN zp;QQL6ln#9kqApX z%RQr~bxF{&gKGuRv=SDG`qI?VO;`^s?`iN|FE^1VySq*p7XhpG?#eg|l=P;qx`_tD zvwd6~9NwtpsG9T1*b@J0T7;!K)UA{FsM(`ko;lmJDH4~iiaLP=(YX$!VNQYAXXi&u zkiCP_IL?^3CR>fSDvKeq&*pG|8Qfp_CXV+Ium+9lVYJ|M!Qi(@U11EMIs9ga_!r^7X09ok=svGRsse}_!Fi1QV{0WS;a^% z=W{*^%Vq-hIIsP^M8)Pod+RP{qfR=U?!rpVOq)eQyc2e5)vk}}M!ZaUiWv=U_} z$U#8j3KMJIK-dcC@q9Zv(UF#7!6KTR9eTTXXj$Rm4%jVCRq~>3hS;5w+srfj3S4;< zZM5&q+NVGUEP2W@E=<~9_Sl}OlQaUrFw}7`nFJdHHOpFgk7}M^baGqFr-)Dl0klB$ z&(1|lbKske?1fJ{j`A|Ei6_h~8c3?Oc*yR+QVCgxk6hBx%caX0@n!PE`Z|g@okAYz z+;_>q+UjP`6u-B~-u%~j&bhKdMb})HVUiN;uN}5xV@;0gXd7L-%%;QsG3&D2UTl-^ zLa1VNyAMU;@}gR(+teREZHlafa03EQr^g3-0K#Dc4V=>gVXo9X8QOu|)9}8Axd1g; z0L)+(@P{(~uqcf-5xa65`>WgKb-1n4y~aCkB{ckMx!RkKwq+-cSr0AVZ!>0HQ09Z` z&~nkB4RlOd}C_Fj_0n$xG z>;6R|8yj1Bp@8k3e>X5U6KwoT9@HOB!Fy&ll9DvRTFo(M^W85XaGV}E{s&f*2_9^9 zH9Am*WP5sG%-1f!)$fVYHrc8?Ygad}+yT9l@j@Xm8Uln%S}d3a1dp`@iV}?s9p&+K*ydcK>Q7($Rd`^Vvy;K-Ssp5a_CZ_iPb(~xM zPYE+x&iG#XpWxK^e+Mp|7MM5x&Kcc%DYrh3(kXL1a0lY;CtX}C)}OE z=n@3p6LhY4%kI8r_0AZ=%?nvW0h zm+CWy*|-lB!#6L2ci}RMS$9pwvB9DBsN)X(LLA_1-!2km~^Gb`< z9i4gnj3SOra7-gIbga9baU^N8rPd*f1m9u2$u`3>t4Sby=m=t)S%^)K|S(K&bK9eFpMSLJ^Y0c4f4qkqa%mj9F+g8r2a=G)&>u1Am zW$32!k396Jd$*8viPDdS;~pEi6B*7vO7%F?#W768M!4~* z{P@yWtVZ22Xzaw?b-=u~34DUSfbaO_MqmmVxuv_l*1^(tMcD=) zGoeS|>G+EbplLS423UIZNLqdrD+=OL0ldJzfPo)C70!!%9F`OeC>=$uneJsq_%=Ok z%va1BfhLx>n|ar*U$H^5Tx$=VEm=x^v@NIIA7UuL)LTh zZVjLGM0uhwnCJKV`_;t5kNaJ{&qP=6Sba85G9DBsu9 z9Gi1MItpSQmf|b$cC9KPHMn8*&U~$FkF;K~t-+()?9rdk*Uza4i$6xk0Q%Ygi2r^= z>-^ERp^?*owh5nJ8}2lWhEF0bkcWdEU$#HhvCs9&1OKyx7r&d*fz$FY` z)KJL`7fH%%{y6V7q`(1)8`*1YPS~OdKvACyt`1j+OFBZXmXEZZ8we?Ph^-_Q14Y3p zv}5b1e4TkwP_z_vVeMk-)O*1rPX>uElau-N)Ih`Kep`U(N5*Hg%UzQ+xHT>7S(>Us56-H5{x&OT%<%k;>}?WDG8_a4H`)_;1| zC)eeaXBYkfrDqu)=3i*nO=2QvVyRpxoLfog?08gGOIPB>EYA7^ zKsszpgq!YPj9TAOBB)qK$jYUQn^wh|t3A`>?wanV`AkCII=kn_dR(QZJ0fv5Tui8C zOEJLe)i0as+MR|vdz6Q7H5;wv0O7dBu~YT)sLT=tf^Daf`EaVU9w5?ZQ6ETqYOC1V zgGX>Nf2DilX4?o_-rlugK68KxL^pf;J|r2}+{G}YDwzQ1_Tqq<`Cz*f*_!q{QSSQ41E?qN8X_cVJb`7vdv) z5ua5@=b(-&0RP>@YdE)ST74p|81wp~Qt;`9IeF-$z87aLP-;iv0*ty9p;qA<7?*Cm+T? zDv>TzF(c6+ZLz$L3+OW0F!kNbQmW;L!4(@o zkbH#4H)6axi?yUhhk&-k=v+Rpqi zjq3->1F=?8T?IJ%9j2XmEY&KAAQQB-d`Mn=f@uKOYP7#U=QdF4imagFv}Q&mU&E_cw?xT;F=a5Z#OkH(?rh8&jUmK~wYV}otj1e?(T^mZYe zp#ybma$T6=ox-ut9zq`iAF<%%^`x^Cqg*(8K^Xc07`lEFb@GC>C8s_(U!NyBB)^CU z)|Vsm+Xjsf_q!Ve&*$0e#evQi&$p+ir{m+oOHFrIo97)Q*Y^{Na^>}8k2lwQSm!HI z{OH}?iqE6-#oy)n;zDom3Z($IQ-;p|o896-l}_Dp-#xxuQ0ZWV=RZnsQiDw4T&l<( z4A%=WhB@k(2g}k;F7H|{{O4)pJhO7h#pizMo+l`ohDTexs=J(WigiTqWwlU_BzyzM)RyM`! z6v_+9*Tm2)3ZODVojkqq^@-JH>bsTLm8SrA#ZFevK`gnQ>cCAYh1*t%bN>%v?-(Ro zutedGZS#z6+qP|=v2CBRZQHhO+qSJY_ulv8DdI&`cUM(r?(Y88v8%sa%j5ANj42Eo zD7i^hv@-DF@{esH+4dziZ73UURX;IU(TX^GEMQ$(`857oHz+GF`M zgqb`)ue=M6877jkf^n;ldmT^Zd3YY1bj3gx{L4HTVKs<{2T)B*3$U9cRwxB`lg(xk z=+^c7{B4*bXO87L!uN2iy1~QI_k`0D5q7XK0QM8U(x|_EZqg@6L?8ys54Vz(?ypsK zRg;IJ{|!I9Hm97*L!55AB9nG9f|1$&U}lb|^ip%tTx-u1mO4(%%yVg*Bdo>HDA&^Y z5{eE+>Vw2X*zIO=U=@dArlqZ?5BEv0jXgvH3kdxNyEquTz%0@P783D>@<;6o1m>a# zu>_RB(lMr4c;|)lK3m(j^FtBXjw=Y_j0|=940xLgq@5@}p==JmJTb9?sf1re|5EGI zo$mTHs99&!kIpR1G=vh1WXkSo8M5c!HNxN{9g?{*;uCX{Ub(Hs-kMRa-*1QcuSQB> zg_#KYUXYX>7?yBb#c5jzP*vns+9I#`*H}9Ldb3VnO;dkd`aFIUF?sl;;;97(x(jI9 z_o4`T({|Y1REX1{W5Z`nUT=7Z(cwqcS0dj%1l^#vjm5HQB2$I-s9OT^4Y>&_nx6h` zJWrSJRcAO-uddJpw;<{*f7#n+?}xH#D7M zo@@N1s^6bh6>8aM`*T%LAIwd@XQ{d^yBY3)xfaLAj6>o3==N>bux9S8mi_91nZ(@4 zN07DvH^4BPF zoheXD1h~pgviOt{Drm{8p;{k0;PFCQp|yVTF$%yKOc9Qm5pgMP1;?yDbYf7Qw6b+0 zX=3osi-t?@E?}s@VS-Iw5<^YZaZxXjS|)&OR6P&K@uZ&ut5XAuJJIM(xgE00Ne>o6 zJe``wWNqr6sw45c9-kfdecqum`53Ob|313#82vdwtsQVL{-)gxnbT+b0Nf&`FCq|6 zVaJk(kwTQiW7P)o{muS8#3X}idWp@1H|jL&!L=Z+d9yZ`)F77!a3h*m`TMLw(LE6n z!lV2>X2Ub1ch(*mlq+QiO4p5KHaXzwYRIumt`Q6hov?TijPb;(XcXp6H7frzke8r} zNn!7Bma0>LJ%L-4tjT8kS!Htj=P}1jo2G!yXLP-%**tr3HWigUq+Xy9^;OwX04W|R z8N|8QLFzTBZ*Rk-$4_WgwT5E>oXNvThrQ5TI8Ih8}Z zWYWT0Z&JM2EY>uQs!}$f*R;JKI-r))wRHawyfb#f)jmA>Oc(LYY~1w$!zE4W zNOf!7OyiA*qZn_h3IZJTem{LS@*sP>NPQ)GtZRq>wkj%+R@1c3;o5z?m6j)1X4Pz+ zZNfIE*2i8lpEw~H!vx;*B=rSl*7kwSwGrz@IC-Yw5g$cJUvls_IJN67<@$iB_rz5Z zrfCCDC9@qYeBIhkc9P_MeEohFq%5W>Sep-EP5$5%LS>~ue(`iVBIIz}-^-+gKh9{c z(5&=KNV^L_4T&|)oMos6^WyP5*7W(QRcw?1F=im+qv;^?KFv(Q7i>;+n~K1YiF=r1 z)THx%1p}l30$MH*9+74P`L{K0qSD;q*omZf0#T5Ea2QFwWvrqhY3K0fQ^Wl(x3Ch> zmZuo}2S4qGQA`Tsi6vesdr-9XSir{sJ}i#-ncRuM0%H7KSs@J+paloU=~d9fo_``1 z&Ha!YgTp=q2TqF1*7;sLu*P^dsON2m*N8q2A*bww3*aHa;D9YMoVXt3?LEO(y0#J0c^O8-cEz?*_scaGg*9QRw;xS15=Daqhv2$MUT z5`?&Z)s8L?sR{}>fwu;2<6x1Tv#jt^@i44}wr;|@1K4t)clfqc5svZ|(MSolKrKe@M|66wb}b>rMKuX2)QO}j1{_Kx(B)jpu;p^8bMFN0zsUuWaZ)EuBCm9T(UMZG*)O8? zLj()(QS~+8mz}{=77wWMmVOv^(JJ=i++-s8`NZe%ortfLLU2=0EM+AuNjC@5z^K!( zKYeHj_=yHkakHtun}o)Sv__E~2MU^NM0ciWh_fsREE~xxj;c8~G*6xZAv8>Kn3;n) z^3?!9mJ00@GbAa+ok`vI5soPd4akCr(syP`h^nedK}M=s=Gz0@NP<~$dAW?uth)9) z0Wou`tY3IpR5vlJtZ(?*R5vME9;W1){e9?I|3-EO>c*#kgB4LKF28%-tpBZA{ol(Y zACv0Z9pcyD1hE(um*$QCU1aGuFZ=nfjLj%1tNMS5k^e3JUH|uDB?p+lZ4yOY9M^`5 z!RoiZsCVI}LtfWwMCk`=C`8{A^XP}_|)ClZfYODT9JuLrEMn*o@N&7$y5Ql z?rWd$s$l5G1MTMd?bO;+s>1x0G`PaMhz$zp>z6$xNTwSR8u$z2nbTG0*HG*{N%tIQmXtPR#B(r1a~YmehxcLI>~eI?r+GjnYy zKDSGce=@h;BE{5`=Ca_s%A{^JiWoi#cWn!pjo-=evc=`1#vw?3=p2n)dd2Q)Fc^lC zXg+Y_oJp`1#XUoNsZ-}%5wWQ|UDhmhSLdU0cv-WxOP;cGa2?RQ5B3_+h3FG$xWQc3 z_SUH7(kuTg*TU3Ouc->8bd@O$<7gHfoCwVbB)$3TjBE1m^gEqO{;T6ldeugsrz$7Q z`%cYK`d5pn#>`JFi7bQwP;LpxF}Oi`r#?Vbb;w|^gY|j(LvzS_?{Cum@iCs}uQfop zWW1G+AtCX2#ySB+p3uY=KCuZG$JX?{$y9@(NOl5C6bjvq@N;#I2OyEppZa4_sG%;K6P0Dg%m!we^+^D0k}%ar`5e{2!ZI`z*M)gdGu zrz%uS^c+RUHNyeRrz|q`{}sXTr15fm(5O>A77uCarBp6uW^_GH9Npv2wy2pNEGs@K zrd%TX6w&z?QZAvUz7WZWHL7Yv@bFxZ=i)MK+UeI&o5FK{9j9hYY2$ult-!*pcJK5- zoD1=?^NSsi2_ajd0`!Z~w%~q8xVq!#sZpuZj=a;_GGpv$gDj;Md!6S*{18fYk|Mx~ zGEoQADTfzWGO;pKp%7ZhfSOVDqX1!CQWHJBH~RiMzJ5Y^4d%7dA+KAE=!vVjRq%$d zj}QWqmYRiaVLZMA$T=C!cd4U7S(t;8C%8(einnir`~I8DlMH*uE|+I5zT>#sTG|SJ zgg;=0s%vDBj& zG}R_9UOtpMX)ma||Eq2?L3(oLH8vL?>kh;(G0CGJ598vLF@uQiB%qOWfnp|{nkYb( z4Ty92M`8gQL&w%q0mF<41&7R)_`91}z zBwT5KLfjAL6`t+YHWj8x$|EFH!Tl0c3r1Y?t{m##sA~Llsiubgpw&r4twD|&8-Po_Hx<(Ppa|0d~|bNLangR zr__kckC?fm!BJ1O!!CSU21@!+x%3)%{tnT$J7HT(6B!N!xZT01I_8f9a_4>|C9P~M zEV|RrwJ{TE6%hJX!}3tNplefb4YOcdi(0YL{{;0W%=mk6|30}IV)mQ=N?VS1Q;q8ebevNB3iqCF6^g-r zkMnl2JZ8BV>1Lb%>gZPR+V>5(JUsKIJTQMo(iz$Y7!%srIA%TbJn0XCt2RB<*f2bo zVs$M9pqQluOrCZ4^|k|w=eh%a**zgGznX?&rieG_0Q#3=xVfj1IYpTl-|*vlhluw; z1r3V}77iK~tgQ#o6V}dSmCl~meP>^~H3w2>wQwHkvlw1U3lv)|;}|@CJ~f2};Sk=S z6UtQ+yk3$f%;>X$6kp$NR7{vXC#9?$CrNH4Pdx$%^V4yKm_E93wg0Ff2!z$Lg9sqe zKFTQIq4gT3PTv2gBl}b+HEn)H$J0z=w&B6{uc+)S6Jg?;1XamS0&J^<^^y`m0~!De zvqrnaS`_Od3Oqa+GLHf>Q(@Tn;h|h;=;_)7bkIIo9J6UUn8Z({x`&XMi>VY5) zhjIcaI+Fh172(-8Fhyhj77sXk#ke#3nb4?iQ+GWBxuT-yty%6H8y!22L8z62`Cg@n zp#XAtEBtAEZ?TZo?Z>?yB5dTvtf#+fDxNq8hXyUxwZ45G?-ZY8#%H5^Yz$p=6dm)mi% zaU*$|M)lERXlr5gPp|K&s1vggt?GMTh9>#@ROyeMBoc&P_R>>l!J&Mo1zMz9{@Kak zm_bp_V8524&`wIa;|gs={#LR?M6Vba{##wB=zflWwqbvswek5``*t3eold2V^qpvC z={B94>U9<;tGWbm<~E6I{hiywN9$CVT*h|#jQYeb%4O!3ZMAmC z(QB~?i3nda^;o@19Smj|34Ucj>-EJrcJ;1eVIUrS`EG7bJ_NG)5_hm2J$%RFU6N!W zt{IkJkz27rkTP`R`P6xwvG(G-sW|HwLLD5*dGU6qyK2$PagBx&ym7` zIod;Zqq7lgfrIvPPmhVN@yTSvA-6-Y=vpAch9dpLl>es9{Z~(=pu2msxp1VokEw@@ zp_UFs=;{0lFwM7*6T0iM7^fImc3&ytDd<_$6%iJWB~9hWiWUn`@3I zD>d82iT}mBDA-vNrC4Kx;37mzdbZCc{0@>6T-sm+Pd9>6? z4OAvfr&|uT)Uu$7;5F(y{OT~M0W#Z3Bx`64xwK(DWN{6a7}20pom#*Xnaav?(#fGZ zlDw4I&`92rQeRLJ9KTA%#XoWK2<4hkDSyuZFH*N%t%@0+9O!(F{-{z9?(eax;Jgso z&29c&qtu*@|E!E+6LCnEmK)re7Dq;Y21!bx&SSVVx)L)PRDEIULS;IU$n<1PyTgdi{P1PBU!eyzQpP=76NJ5U^e2X#kGFbRyPvbxd-RyUOz9Qqv`x41Nnj$YAn?2bTCBL zr_h1~kAj@o{OSk@ISioqyr9fcq~!=E|Cc>z>*Q zDHr{RGj5U=ZLnCAZFC3{NniU=+@eI%Cclq9ub@Ep;By{A7h#kLKFTr;(-f3eth_CB zQI(6zGcl~_lBwmvMSbD^S|q*6+`XqLcV;S$SSB07`{~CbKV0GdA6ER0sO&EgxcvGlLi+BvyCH&DU48wN1gZ`yW7>@n; z(i|!^=oqLwzoBi2!FHMxiIsve2kee5#xoYZHOThQfWdv*T9+>aa7OY(SmHT3WcIMe=q5c?+d zQGVf}f?{LL{ubbYvQ`h;f6K=v%@h&a&bHEWHX>Y?rZysF%c8AqW%$42Rne#5VS`spi7kU&HfE>-Jxacs=ttPe#XRRn@y~izGRd6%dk_o1w6g?=&`PmM?e4)v|>CC0_8J#&@ zG%ivKw404Q)lvwyD$aPImth60S^+0^vPoccel)KM0bF5~amDZfJ&m;x%>ValT@tyfW70Nd_7(}H?QsVV%P>SgJ4l|cAqIqsBe5A4yz0>^NnCn9eY|&D-NY z{}ak?@=0_PQ+cDNiVuP6^1t&w_^bCHJ%l(FBy2nix#AjsAxL0S=!vx5WUwVs&BG^I zwj3L((TdH&PISSegJfi~sFF~|q1Y)HTYBf+s#=z<^SdO@vm|j_V2dW_F$WL zW!)&RYv*}nOl|U0U$#YL`rOlomZKh}S&oH}<*ok|FU=eL_a8GA%8G)VlG}lt=pNqh zrLTjTk&$250`H&mjf@HOp4kIEU&EpE>CcOmf}GI^J;E>?Jm0P?;))8M(!?6`JZk2t zWo|k(SytuAxfitr0t5@ z2}UWIHw(%|ztEPPy7=fu@I)hooCyXQf@>rUUsTN10Nyf(skSnAog4-XpJyD6btgAY zXBkpPL!-Vb#iFdGtiF>hIi5^DZF{kYqT*rozjW{e3I!4|0fhl%TKatclJ+#KJZEH; zPT9(T0O5TerXx1&M!X5{ik3WQ&Be}Ee4dGkGS%NR(c~E_uD?Im5lR4(a>DXuYw@2+ zG|d!uFNZ{7$g_B)IPDfO_znkyR{#pwbf(~|Cj)>!sB`RfSaga7Lp#;^bT<-^YdETM z#$Ym#O7GFsR_&>o^nsVdeac_ypjvf`ZJ#)*`wX)PqR@ELIMp53KtyJxO*_Bcp4f(^ zv|oKV?>v9=c=~*bpD-C^YVisUaNBiS0**YuSEw7594Loi)xl`a1Zxg{Mz%lfMy!qA z0`x$|c+5zg)sn6XJ#}OBdd>=(vl^R?v@AKsriZ^c$&plo5ux>w3lGOVdOrSaLi^!I zfb_;EvNH{3BBzv(N_}><`ueu$w^Y&-0+vfjTUy??Tp{Trv}}%*k+0beP?kto;^3cep( zw4?gqdhyG@RA##cfA2-iwq;Vj*Tzss;0|>mDNs9R0xe0Wr)jIL!ZR$Kqfj{z;~F)_ z(cN-r#Vo}7Yc}Jf8ik9-FHKGKI}gM_yVc~%*M?X6zF7x&oGC6JUOtYUy=D88jRnbZ zvb67bE^?qOQ%2xEzM))bgHQr~q#TQ@5M4r~awlUd#4n;wwQXU>RphEY8UC zWlcj=G&nkt4Vl9ZwBg?{aBX&3kzo_6mj^4$S!>)zt_;DwV#L8F%gwfd1#w5oi$M8IWLsg-oWpVs1fx9UsFNs-G$S6Lp@<0&{yEe`uG(YxrG?A_ z$Sd2nRtTsCJKGURZ+7&&r_TRq%_UkT0+u#aYK~FL@1oRDe3}rcG!}hSC6y1~x44x< zmp&77O&=Pw%8zGi>(Y$+j)IjN%$n3Wx>toS9SZkc6aPuD<|tmS8mY?d!(Of@>0!zYlP)`&dDm=PKOAqO)0}TYdH{>7<75La?Cl*W-eLy%bu9B3gK(kdS)5tGT5It{btmZcm6E%(B?h zTF~MhYhXW`43N(%^qUkI6iqjY-3*w7Y$nUE=3y73F;UlOP;1b45j* zav3XU{o7H#;V+%g^!%A&8(YY7zg7?s?!12PQLJ@aP ze1sGHSQ{T#$JN1`t}Km>3t9P$-29qGw=*Gy)}wEe^ix+}d%B;$gX!5_Ys zQ|O>-1T?sEEB5gd*`OU&1siOX_L8Uu6{$P^?aZ8uJH4QWDxE>GJ{(>U{GG;HIBSAS zAlYgvxPBo3;F;Zb7k_VD06&N(zfldM+#wFCQ#c|JHElnV0>zVEyW)p#(SOj6JSll%fi;B* zUQmFm_D~-0WV!cT_HG1 z^G@#r5t2@L{4~z5<~$?V5TZ&%T8|o?AK@kLb`M$r;lC(Bn1CX%5GDca;9y;Dp)RtT zz-rwW1)|bQ{Z}gxR`ju^S6SCe-Q{8WwPrd|jnQ@LLZ(66*x^H%j`4G+K6ZjktK34> zysYoG(!IFMu1?qL;o%kVUO->fPHS{riW~eRMBilVS}B}kOX)mA|7Rir~Y3MlpQJ``%>*LAaH%>8UXrsexw+x>z>$@m?7m zqP6Q&BT?nM221$|2QqoagAA~CIGesu?0;Kod*%RS6Par)W^FCf>5Mm?dWbH zA*i4rv*cw-lMbzTh%<}DtVMOE!uAmGyOUHz(na8|&GzpND>)!9oZLM@J4{(H>+K`z zir0m}DS2UjsMkv`OhJi?C=G9v%~WJrQEp5JGsQMP8M-q_QC}cw`&ISvpB+3^4c)+m zhPqkT#a4_D?9x%WPdGMquTummh6v>QHQl)R#~RYErU(2rNF}b8@B(ii5QaPbVBkLK4^ten%_t~A^vZ{Q0MF&f418T;9Jy6`2rBZI4AsY^r3T}uy|tB>SXQ@4|&kN$sTNwK8vtL!71#@(LlIVSHn->d z2;DbAY#?3{L?kpN@JkyyjqFwH{^xlViauI+g!5j_nEiyL1wm@F268x0W9D`_jp>GX zO+9W~WZS@`B=bqr0*5qBYkd@RYFx>o+#%5$GC9K!R25U}DC5A`{>fmioRhc=%G zZB1`}zXCq#rO^~6&1R~-XRHJ6y9hkDf9k${Te!4788yB=6cMp~K$F{-TG=_9siRZf z&4#Nr3t5=_b1NKp z!|-PxY7%D}(Tz^PDw;N{Hje;OmfM>$4)+mtby}QO=%pA_Z@{A28yE~?Z20MykQS&L z<66lDiKY|`aVhbZXZ476VuDL#=>zuUex_AsDFfSzR+q8%FoBMU~-{PH)-yHvo6|(K$U}*LK z{j&diH{cf#*`{!cZ{&`6&>0trIK~@k30jjsBqZDyk3oMAF9ZbIZaPih``ViMQ_ZEt z=d)HPmDg6Acc&Wh`;_>mcb@W-ZN?mvRKFr0kNuNPew*}SEuk4Bp>=u4G}b0ZldVmf zA|U3*&@kk?`zy?HA8u|rHjHUHl1=Icn7U+~#V`3aQnogV!okz%?M3-+OTVI(( zPT!!3y|lQJ4`80{f!-#Wnl|sPB4u`~h0nw5Hd^@8jWg`|Oi9I%Goj@R#a?Kz43gXmB8Tu;gLL>L99K zcgitK?{VB=m>)iH?U&J>K>l>n=6`+bEpISeOPXZC3oTEU5SicfDp+C$u8>-zbqRouzqI5y%auO2irE-Q$gVzx>f!v13+d zB<0TF3xQbU5JQ%aREFdXQJ-!tmZ1PdHNlUFG0#Eq^>K^++_d#uWx^3;>bB80ni*`( z*RgO`MB6>72Ca7By(Ng!0^%o&gMg2<7$!?3F`$s9X3@%oi2dkmW&E8>Dh+?oX0g=K zajZG-vW^Pg7^-}sz@@0Ar|}>)++g3t(kpXpv^=w4zPL$+rQK0%0}ukP+^2rSmpNI8 zvhdrUOrFJRY6#?Xki-|6nrzjM=T5FNa@AVME%!kmbh}^{MF+*8thrn?+HX(0fMgQu2PrNYYue z2q`oqFzY9=Qg1K&oS=_rMFI-LPkt#|y$1GxrtPybY&YSs@a!}( z=5-F&_=UF=QG1U)oHUNy=r3N}fNnvuI?}#U4%#+If1>dc{kqyd?GcS~vA%(q)5}XQ zg)?s)2Y||j9S4D4wF;iauzDj6o=XGYR;a5LK8BvVSD)Dn4r};A? z;`em@XXJ^GR=2y`OK5vEeN8b#c6~I{n~fOxY=Q#}O_8S829)8QFvXSfbevx{pDYM8 z7KTw%kvt7U;r7{Z>kE8OfBb>WokYs27S`uhI<8N*`w|tdu}zWwrQ&*G85I=)9;w^f;iGbg#kJMM(~CQ@$F#ED;V5?X58kw3noWhLe3C6Nue zfgEodKP}rlSM9ja-rKAeI-1DyhuGY;z7YJl+fo?4uc}C(`PXdRAqgdD$$K8XmP+*T z;8#zh-tAtL*cc2SoKsww_d#<`Fv>0sWg>{<8t#bEoLes^tHy+AzQAD%nb=HlS=3?C z7C5n3(iEghKCZ6e3Uw(Jix$;5OQaPxxWDHzM_AD&eh3VzOV zQtG#fseA@kd!EgwJO56yvWG-S6k6VvC8JKLPYa_Z0dB=ZKnE_QObmeG^w)b31#hR0 zL0`;^rnm3^Y!`!&wavK+^2dN!mD|CwLRm3T$(PtQe=k#O+Eks#b0W9l3rG=Hs2&{Z)m{}(DbNOrVmp0cZeeaWR>mr`=}n+h%97Gjz;5LGhghm5RV(#=SseuP9E6Oook zGX-k*1c*tPib0s75h*xYPKm<|w_R2yt^H$%{tPxu#f-Q;3lG<}dBG|JHCP55({;j7 z7@y~Y`VU`Bxk?k!>+>Af|AgF*>Qe3xLU9_8N=k|pSa3iZAR*-<`It&wY_ivr06O4> zvOyuU-*Aq+$zCLSy zmMW$Njy8aVKz5qaWibwhki<^^CY~cgJ#uDfKtYN$w z`QUe{WZDR@dG~4@{~h#j<_}BwUJ|m`agKTiGW+n(1Dx6#3yDBcc1=O)iq5 zRw~DI(KN}mXldRn0BgseNE{5P52SrfQpLukSO>&}HWl3|kadlM*5GcQSgkYVGHa#p z1#J#c-w&mMuXcEd&Gn23VeSnW^?Ne(;eVriNpArYXy(H}>l?u101Yma$fkWrIx~Zp z@>fph*L8;y{gBJA<|TE|@m>Id&CNKQ{-`SnN2^RByfuC&FVX_m>7iDHRdSKFzQ>>& zwHC&?<3tnmW*eR4BI;-+yadC3ju=-dMCMO^xmQe#CI{!jM)9#>m6vm{=;vMcl8?Av zUB$eL=~~Je(THEp2<|f3uv18>)+>U{iy0pRG>s(BpndB$<$M18Rw)Nf^#naP$6DZu zx3~A4H%=pN!b$(GtKb8T8YP-bUJY0CrntM%bU=~)ad^!Va>MFLJGi~R6)hC{B1n35 zsgA{OZ-Qf&Usx{|lD$oxBTD;#pEJ>;GT*911E0#>=K85q7Xy?9ycyg2T_1zc+Vfto zh{Ei6YC0J0`^qy%JX0j134)pMi{YZ;G_d6pX$ioZk$BEKSbRkOERh&NEnD@3TlaDE z{XgK^L2)e_19zUVdEsv7<$$@3*i+5*aZ;^sg%$^>CEeAsSi@_0+mDC00BxugEF{wg z5bGRX0&FK0OWD(v8UMSy;P3+;>|ZsN`Em*EV1_`ZyQt}e7Ho{1sla!fy3os92F~@K zu6@@)FmrFI74oe7e@;=$3Bsjul@sJ0$o0(3_Gru zI%8gRK<-hF7?PnvF*&xk8&la4If;hp^b=FA;oM{mcFSj<1hs6_?HnbrbOaY~=k; z%sPXA(|?EO$}bmOH!ge1sk+9Yy2kLo<@77hZ;T+nNAvddKlQ&AC%;8Eze$kVJ-m0v zzl;B~;_E1=u;HC~lyKec0+3NpfmG5}%iK{c!sNEFDOm})T0NmU3X z#JbCGj7U2rc#&?JVXu0}12TF5&=Qw_jIE*;X=9&D z)m((5C-7HwjQ`(7Glhk2dkXRpzBulTIGY_p*nCLR^li#)V%oLD+z4tG;Nnwrr7H%e zSuQ=5NwSLYB8e#IE&Ye6w=mjH_c4~*%!3oVc{7?Q3056uYHpcb1bVN<9V{l~K z5!_-Q?rRWuN-Nmy^PXWLIV<4DO;hY0O;DYVDEbRwyO#QtvM>S1BkLiB_*!|8jwQ_KJ%y$djVO62KyiH4|+ zJEf@2c!0$Qoa$;(qkp#~<6^sq)r?{Nl|_5MzZQHy)ap0S7Mv6F(XcEv7W!cg`vx^< z5Nok|W3KtdjDT|=#oPe0z3m$=!{Ou{Ts{UfTHFa%BK+6ozym?yspp(3-6BwhS?}TQ zw!u`@xrJ&IO_Xw%Ua7k34F<7>o*$gRUX=a{)13WzVW@Lz4x1m$$F|HnHaMspJ#WyP z%U7SjJns4wtzOc)D=#-*vi1~UQQKkOCk08`VXcU@JZNgl8y5l&=NLIgfV|(cDOnTS zE-wh6-_lj~|9pvNGHCF7s`a^heDNK$7Bm+5enr_=Sc1Vsx>fepw=q}vOG9O0Tty=N zIor?crn6!$#ja84_4CcwxXC_^YB?0X1j9s}S?mH8I$O496B}O-FY9{y=0Fu;I-y&y zm96Pj{SeetkD?6-HD4dQ20Qla(tvPMM3ep8t3vaI<|4?~+(_M~Zq(@|a#PzT@CEnx z^a-wQ0$z0q2{JVL-ruJQe$9Yxj|t3Fu0NO18Z7jJD4W3}Qa_ZJ|7|!iH(3^5BsUU^ zgrovREj`H>oavV@Cfu@;ae29sm1G%ri3VZ=4dnQf3qoQ8;0GhKv7pAt2H#gKlDY&) zKtlt^mj)`gwz$*eeSU5J6}7SGrlwzx=R+j~OnPm+I#5iph1^V3xHIe_)O&4a4;{JK zMc<0*0S9B?@c#KI@XX}W98aPnb^-PHzUjLtSGLrT_X&4=BOWRx6nI1f0Kl)9Ycvg4 zSaptErNTdoyM?GZCVB|nh`{bRb)j3ZBi^S?GX$|@(EU<1;=S9|d#X_qMU!I~>EB7i%!Ke0G$b4Nvfw%!w^~wIcn9 z$Cb~sB`zLtjv|c=cIe2L4ufgKR6@T1kke3c}=>feLHL7B7IW za8&+U_zP5oJYnHAKBoG7Pw>6Of)Cdr*XrGEAyoRfWZWe4by}f-+Q{dITqM%89$(iJ z2;BIRFLNCyeP>PsdV1wm8t`?;&nj6{jd?YR_MW7(3|H(xq(+0J6ttI`;`grp1)H+a zV+CcP{wO$OB0LFnrqum=1S|VPtO)Nj^zCW{Eft!F4@I&rwvSy(3UZRE>ShCU^{@o0 zi9$TmdPPxJqUr%wp1C3zt+E_^GR@zeieCY^we~CTCfeGwq(XXX5+4ZzMkrET0E%LQ zWXdLlH8%c(d|QgvL2xAzdTAe38*cDAMD|0lx<}B~w2;ifdNX_X&*QQHw#wfZfO*DsHw>p>pCI6pwwpr@FvHFro-0B>|ZFtvvod zxMRBzLePvy!Eyr{RKs%O&!FkSh=AnA+^p?K>}14kK&$%Wdg&~rSEcW#FV%D;u<$n^ z7uCObvWpU+q~RM+S8D*Ce9YJbbOjwMFwxo;!rHXQbAUybSqOqry1z|9unn8Bs^GVPtt?#aVRp6 zRaBTASwU#4;XJXng-p~p&4QeM5L8YUJN0tNb~_`UveV=JhE_g=Nu48l7G+^!Nee;q z=31$!9gex1I4wT#zz6?*`+;$ISq zrdZjh1bBbe`w<${US~Qi&4S?EZnc2&juBANs>;M*uDUZ-c2j1h=y zGgUXNw+pX9@ihH|?^n?*(%W~>XLwPCLF{MPYY~BF((mC&nw_u{4^P%ejjm%;tg#SE zpbmPvG#R4j%`PBg2{%3znlJ>xLEWA+lhQ$)Z$i?NKEP}dEw9eVuXRDBY)mnAWn}SM zU^%vH*}mn$-`{?vq4W5zzqs0*c(=>7VZzJ{_%RbaV{dB?hX+swgdxG8^>j|;xd9}y zHJr=me_Swwq}mi=g?*;1Jjfk9Jan-_U`5+F?1f@ zpe6dZ>CjoEB1_09h(8MVQlqQrd#%<2Tvd9M4>~WE5T$c1DYm{gi9^pBLq%^DBbfzj zsQj&ckHd3c#!wstgU5NelJC&|v21BR#}wDU!2THpuB5+D>zt>rInkKz=_+O)}9tFeOLmi zI%yuO?`O@;9uq$?S^1$tN{!pmY3B?SX1z6ApyO*Xa#rwS^8!}EVPYLmUk=D-thCbB zP|tFWaKj2igUJ9Nv2TnNf`&z}g1rj%q)3LQrSTY(CS2i>8M;ba)k}&>_fngf__#qx zt`MKwY(sHb6RTh3D8i^f^b(a=TZ+LLD7&+SqnEvgy#j?23LN6a|4DJY{14a+i`kr` z!$`8h82DYsV{HGoQ&?ZFL_}s~Qzqb=F5Dc|u3s6++g&t`Zzib^*3*-SKoZtP%#N`m z-`FE?5arNHK59$#q2C$Ns|Qu-X@hRpdn&5T^nDFwn>UGSJr>mc&Gk?PIVhtmfx*?3 zP<^(~wrV+x@vQEkUo7n9Xr9f#jpL9v_^Z?#6X$||&V)p$$vt&)0VF+vH`2Nb*rK@* zhvIWWTjF8>1q)Mbq_DG8E~<*UaVgSNq#n>bD-~9L#2a{w%2Y7>3*X(CQAuYssJ^C? zgBV1xsz4#Dpiau{ZN7F1?PsU+%%!}fodIfBg{$s|mJF0no7Ro@%hT4jqIJDcZrHVq z^~t97eeThqC#w{a4m)qh33Wb1V!sYJjSc_ev4f2sQfZ*MF0Cp8`2>l?rZN(?Jn&OV z(+Hzf8qHiWdffo^kY=6Jr*`x28Ji;sAe1zJ=)O*=;(&YGVbALf|89oRe|C0hgm?6|G)ttK%uW<;aAP_# zOKuBUK9HIV>7WCHcm?)I6OUt20lr%WW;WJhy#_Pw}+@Bz^1mpehXJhO9j zQP?7UBec8F0j3>oq%7ZVn;Ko!muO4@X+Bso#2oszYS=KgZZO<@aJj)?`cZyTm{KdY zG>IPi&fTL~e<{)Av6$X{p>-r#qSMY&=8J(f5_pd*5_2VUSB1-;BU3r8b`d*}H;dGR zbXyt6q^%@H;wJ?c#zG5&L4(nd!C1gx=b+LX8cwSlwi}u5Kx_6%;mfyl2&o0x)&1?D zL5!$0Y^WqmsFKz-tHg8izt2AW?$iAI&u3>pJe~jB-+y=ZVF4e0d;aJ1Pxg=Bou8kb z{qVQfU%$V4^>=>$=?BqlZ}6{AXFsr4e@`TGe*Wiw8EMc#m`z=&^BX)e+PoCKceeB+ z1P4(}r;#ftQq$C)X>n{Su;SKaxwWGR8UNMXy{vWAs3!-RYV8SmTx-ANS~FPqd#lZa zqV27=QEq7^D$6%PEOOMjB67>?+So`vHa!-+%io~=`};eK|7SUmqzj6yZ zMZxPof|zX9Gg25|1dd)ui5=rv?>9YVS8I3|Z(fe&Yox0@7lGzrv)wP6pO1>_h7TGr zJ~bnh+_2rugcK1Q8e>(4!juzagQg*SwI25SXjh|F&w=5%VLOLy0OO*+a5?jCD6pt? z4B_&giVbn2mfUD?9qGEIg9O^>-;|kW zME@V=ur0{NxcyXL{u3rc%qOntYH9zU4Zo#@iHo6vFcjcNh|1O8X*nZz8>!F-+Fzm6XwaKIB^hn-A(DvDUc#g;J1^--T^zNz=n_ALuLJG{thjKCCXlVr+e$g zLk~tTq{E?r3`>{jqX#|#R^d!mIT!1rM0J#?Mu+1lQNeYTs79}&L`9Ag)ztPKC90!D z^*0Z?^3K@^PDVs0R2=S`)7jPA%vcT1L)!&*>|{x zniq{>3Vn(1FIlzaXl`F}Bg4LZcWRJYtGju1H=obvrm!)TQyBB&_#=)0*RnS0KD`mv zOrU+S8po%lVrTld&?h>xQlASbXp{W(AVGcx#4AX(2)OK_z^TaP-B`RwyedS@(P(zZ z3s_(wRO7CTZs0K&w`5C3gNIJvFn&0X4YduafURy~F0S;J^MY<* zo`GxY9lX<0PfDo zm?*Y~?Bt$k!(;+$euV1SPVm?KLE)lRBQBCTzUC~N@zqj4hau>qu67U+xraCFN&}7b z>08B=s{vsTFjw3Ka@zp-*z}!?{`tDlqa9$b+(2;mk7YfL`v;{b>W|U1b2`wLz5H{N zNJQ&+IMX34(4Y86;&rez-bmO335&x5x2sl%Z#^X|fAGVA%*mk%jt?lo3Wv}h};a{-r&FGP~*LHpSb=wZd(J$5Zh;3G}ER);vs*DCfpj@y#v6gtI39`-eq zwBF2&zYIbDz)^|f{;BdYMD02;%?%K>=|*j9;3Qk&zUPXZd}GJ>#N6}d?fY=oZBjqr zZ(_RSt_%QRKP(G7!Iry)fIj*gE^>o0nm95JJQxL-*4c<#k5=qC`5=fJ9x`v&T+sVx zGtyGl+&dI^Vnr$;q<>IOvc>|taC<@a!Fi+ti z?&?|D!DN#n#9p{q2(Zg5Uv_QFL>hHSkMOvX*Oe}3Fx$qLMg}UwAUw^Ee0}*cWLvg& z!ZW^Vg~M}nDWNN2%eG529;$fm;{ces!*pqQKCTIli@_bqT}aP?=*F~S28bK{aJwnu zYN(H6y$SR^)JH@osWlHkYbw8GGs13}AbhOzwGR=-g#nx}M=`9epH&SEdbv}oKF3pu z!tQ0YQ&~OcTJCxS@lNmpdLceGvGP z$zf;kJE6JpM}}l)w%ka0L(Xh+;F{N)X1U1Z*2@!|E524w%o?5RR)@;}a8Wq$icVEp z+px^F9EKA%T~3yv3O)AOZl{4=!BpK#XW5^I5T;giDlC*?A9_~b8R%kvRM2E_uZ|go zTZ>7Fz4>eVgkQRjuR68FgE*=LMIq-HJJmFaA$sataxaPknfw)0ZE3kwfHLUw8+yyy zoH!@;by|cvct zs{aPBR=(Z<%4mWVuhrED(F@OtJqfFI&)T2k0D}eDY;-&v0&`YI0J#$qtS|T-B!Oc z_hMnrYl9Fj9j>%d{=8y10r}OFUipC6P2RGK{OH20euPc=<851G`90f~1+Cct%q?CM zp%lz=cw$NpKlzP~${HTmxU{%sIUlMmH9WR>x1Y#y2h#1XWSF>sixqDR*@%r5k!Ck2 z8haK0-g{BHT#=I#TdR*z5jqm>-ZjS5Q?K+Nw#PHM3tUn02>&xf;+)8eyk=Wy;r-?; zGG!!93RCz{+?^Y)QU(; zxq?D0V3hqCw#jFQQFI_mQN+ySj+dm<&)$6hqU^8KSCk8XRs2#zA&Y^ z|F9rm$;uQPyG4vvHD+glNI@>vXro1f&;AY0*kPc=ubE&K&w9NZvu<}#5)WL4Zq(=Q ztX*Rx=6gRKdiV|q3aM_+?>!?JAd*8;1fuh zu5D3m&3A`ZR+t&E?UI4B$4>x(0C$Iy_}z;c;S08~f5EL`BLGx`f?lr?j@MOf1!e}# z_oe4{UxnH2bg2y5r z?nw; zAnYSouy1ip19hg2-4j}2QOtBx`gugypWg~u|JE+&NJH&|wIuXba$5)-$_Zf(ckc2# zc4W(ymfAyl_O)e3Za5X1OO}PqZdh*0Yhsx(`-ZFUUWC`aQH{_*4G9-oahTiWB|U&# z!2w%)>^ZbX-I&eXvffaP0>Y{zrY@2&Tg-v4hJ|L^Ys~=NA288sLEgw}ON+u_;2N<# zZ1`F-B3Qm_W&ScQMu@Dj!P(mF*5KJMcurN-^vXVDUkgRA_?IH2D8JQKM z`5g0cz@BC@+(KfI+>+QDxg}bVykY8A2;$)KHc>zb z;T|Ci@d3tis+-mt8vYXj&rC{q%PLbU>j&%E6C+|z7)Q`JjDM^xod9E?1s_#2QGeH- z?cpZkfyG44E9crE05}c9+X!63-GPo!#CQo-x5<*`9MY`KNE$|Qa^8;U)R?T(!7iGK zQ_|Ed&?jcN+i8m&UYu!e@>=HcTl>$iIn(b?eMI8Q)ZyuIY1ii7^u(s{>L7<||HYL9vXP4|Ak zm2i}HNH0X7WY9q0JL$C+N5(_@RW5x7)u=O#? z!d_o>EfE!FVH} zj8seNU#@ymDa)p?Io=KnrQSe8XBcGTz`oKCcYw}O2yPE5w$b2$i?l5rEnVhc?)5F> zB4+|ndn(&+aIlRi7YVO0h5W4a;F|dD4(JEI^!GYF%$#oZJC`{`)87{iW4&2drt~VfGou8J^D%^bhToP@U2#tsHjQKSFR|Pb;V9zlsLlfnyHy)ifzo;Zu0SBTF+x(;6demEwdTaOkudKIvlFfl#bW%~* z$mVtSz%A+}8iE{4Coxijwm?q{qQLL&=x!k}7l4bkwSvF1Z~eNW!H|0YYD)yUXg#y* zZ5`>!Eyf%(fH9@s)2GV#w)O8>#>zU;uulBtvmnN>Z<@k|YMoXIUdS~imq$=njh19feR4YD$>;X>r2XxZ98GmRgF;tPZe@#JW z-+MM38s5E91I34*NF9dI=^WV3IrDQw7S2EWV_XYe^o4H!p&;5GFboW ztX@}$2@7ih@ZdiiR_!oN&}&wNHI_nrGF&{(UFOj$`VPEenqoT<+5~|kqT!CmCX9rv;~G&q|#acO0Rr2fhgIl zk`}3i*1rxoztbwE^{>>*X_GqNtFksyNo^9zNY%ATmDT#!L4zHyyw<;xE3r))m~Pd! z(aLO-Xx8y6ZsVYQ<0KQRx;;!Y_o}>2A{(vguVk!YmW-C$L_MAB)s?_8C; zWF;lJ7G5I7sU>Oq*JEiE_X{~RmdF`r_mPXnauB70>13F5`%X3T-rx1P;2JpKfl3vC>S7wY8$aZmSAX4J`lH?V?g=FbA z;X_s?b#$8N!sT0g)fxAoc&m!3Ql!dvvte5`DVwU4Dy_{_?Cl;f)|gx) zn_7q2ooV|3rU4T4Y?|jq7f!~wI9lwC6z8)XSphv9auXhx+)PZrJw3&^4$lju)d?S# zGHtbXx(Nxx2)F`u$hvT!iK^;o(jsw$NYEO*1)sV%<6Y?b9CtHSV*_M%KFxx%so;5B1Op>&${uL{h? zHKT|nE=~TN=YE@E;(P*CQ;?9*cKGbDKHQ0d|F$d(PMCzo_2bxv3&pNWz}a<7vl-MG>^huIkYIT7u8^oC?KozAqpG>h6I z7+4U-9$wN_CAX~Ju+bO?4^YY2tr=CEshZr;odu_(t`I92c`Xw32+3~Pf1BQcv4WOj z&j3veDb|YTV?-V7hC?FhEv192_|TGnhqzCvaDo?Cm*xq_TR_aj?^sS$wqaX}I!cfu zI0GrdF~}8IzfDrmHLEm98*!6Kam%V2f=kwd{}eo}qa1PtgkxPKul8$>T(%Jy-hW& z4m65Y^L4{>_QYEfVR<^ynpY%Ps(;4S^%^bEhRxovo$|c8PRfX*P*~6vS*@u)wtz}T zYqpk^{_{*_vNUrElE%54k_hG3bcbeVC!#2fi3bc;IIUZqTA4QhOn030#I=BP_aMt4 zo(j>OCo~QkcUU9%KO5GTKyl&iu?}B)DpVI~%boEvLW_DMoAm~hi&KM>x74Jp5o*sv zuoA&hh-pWdLDz~`wA5U=;&J;S7@9lM?80S*VDU3v>_DgzxrwgAo<*bZdCs<_tU-B# zCF_S6p-{=Oz~PICux&4Qm``0@913l$IIGleU`A6GT4ci~ti~{OSs&276|yY?R<@xP z&1zP0Rr3rHgyTxRSs17v0gAAm-;Jyu(qhv$&w*MS7}i7~Ar(&SF`&dgM;NgrPxwSz zLCpJ`3LEwqkYT?7T-a2oFdxeyCK;u`gmn`%BNU?$VM7355t?Cmuq5!W$QwF_)I{K~ zXruQ5tsU{?|9$xvAPKvyxDBLagp1acJ{iVy_WD8N;8Cj{ZIM!>C7 zIv<0+>V+J{l#Dp+)gGu-Oq_^7UX4gdF?lNhcQpcL72_)-P*-U{RuSU+Fjr|HRx$44 z;T%l?u!>C?QhW^Xs>cAXdIZp_iGWp}+#l6L<6u?ynzV7Cs(l!$B<30eshWa@kq%Jh z@B=34NDl{2b#PSBQO`Q+Sw}r9wXKP8PmcrjGy&#mGE#dw#8Y}=PZGRS7q}^bY6RM8 z0JJHAXb9G65U?qM<_M(Iqk<6rm$oY;0x~@k7}FyGF--(vO63>H08A~sQi`6L2ENpV zR!WlEeqrFHbSWqmaOu#b!(?!o>A*6RL1jK`K$&}k$vg(A(q{{(Gzs@E6-ep0#(!OF z{Gq^-CIKZ`2&5FjCJ7|TgFi~383#xjhdxRmKMamE4ttb9^0Q?PsihHp3ALW-M% zr-LA+ANlBo{C4-G;JB~!c?^w31up#va?rorz_ff`DM8JYw! zbO7W^O5Y~I3LUEXOq!jkP(lVID4`jNFhctPK`C8~LJ0N2fznLLA^4y{V4wuD!_YzZ z5~m(onm#<3Y8nX8WB{NJyib}n@)rjGN!)w~g8m!;_!C!XCWHONQT-+Y{UkV09tY&- z;Q@Z8gZtp1M#xS9_8EokiM(SN)F%Sj6Wx?uK%WG-o(RPrm`@s1PlDUU*npB@dLktI zSwhJWJ<$P00G}!FJkj3sk+`K%Xr9>i2jHG*usqReuX*wLLh?Kw9M8c)H2jx?rdp4yO5@5P5kOuPm*vuTF_I76TdyG@^ z4%~`mjNdXf9a<0hhLKW1_=JP8+au#sN{+HZ$5P!M(+-zMVGf-B(ZqjX{BIgb(@ZTD z6|6SVfj*4EhKjt<;#N$>JPh~YWOT+6%0s0EYYfHODgKj5!H4ipe#?a3QVa4oX5x8c z$Ebw(FI7{K(<&!_b9>G@8Fr$kAFWk4$?&K(_BL(zTQM&yw&EWVKSez!bE^oDIi7AN zh9%zrmWeFLZ^?^B)zS+hLJMc3*kLY3*Gd>SCsTbmVIQfcx5(u7$#yr-;I+=<_u(W$ z_oAegO)y%@TW2VlBLavE5Nd>LLDUupIVT~j+MRCOGJ^@#o5l#u_Z1ZipkCm1 zt<4IPxYTbgrk@$M728XEMcBuZqQp%>Ak*i`R?A~9x^V7GrWuX(f62V;ufe@v-ewCt z^%f(i;}6yldiwXs+x{2%a5BzIKAijvufuVFlvnL-?MHsgBad8vt$sKe%6Ely5?0R$ zkA}u}?0bffp_7730qU{PN)q(QN=gx$o*eZ-kPDwJn$k>0Zq}P636mXmybb#;cBBzE zLf#1{yMXp9tD%!NcEI@ZFl;v??pmSH*k=J7gkmD}TEx969tR=wuG#=&0A3sL%&Jn0 z8_OaZZ7sc{z;Tw2R5$B&=jyu`G5Z&E)`F*j4cQ3zw;-^3k)TS+jMI2D9L$xSd@y+% z$9_?6sFy|TOjuxdid+AJk?Pi-wFpa-_jO#YD=M0ThQ!OgUH%AqX;KjCv;fpe!Ka^X z;OV`BP9sXEB6UTx8`UwZM|C+%#n`#OdtvN{dJ}U9y2OoqR5_0-=R>NTZC*<$E4}_8 zvwj;5Em$&A&}wb-V?1M}J913Io>4`Z<;1bI;L8*aQ4d9JySV<@$ePYfIAIlMTIiz0 zdD`Eu`jFA*5SZsvH!N3X5HD9lG=14>GHqIDsouN+&U5H6$!L)^7(IyZojSH&7862P za)tR)ET4_^Ldzc!s)mArJTbToLbk1H=d>dwMh`ECqGDE-b!FOKVAu{Vv_uDuNO6wW zCh!G;8e6s+lt5!15htIfNuVfw#a7s4QB&`c2+@h6rha9r4xTJ(T46FiTgjqkD@;a+ zP82mCBJJ^EL`^%oMDhOA4VSmxBZv9?q)?pXt<@dEE;#oJrL6c;3&C#_5@nEiPy#SL_w3uAiREt8V z;c-p!X?8|yFvz8CjP?9Ry=^8@U(1|LjmAD0<=a@z#3|_>1y%J`C70|yPrvhEq$h;@?=9-JaNn4Gjc@db=i1)!uNXR^#Ti(MAe|UPd@=-I2_qVl!1vNntx2t^VA5ihM?3{ZNZiDK z$0kFD0*8lnyaa>8z6Llv0#>(6SO?MdydRT1Vd`J+2Q>GTjI^L9gy)qjFABpa-I{Pb z+}s2c+5VglQ#EW@tAg9Ijx*v(O8hkXqH3*aaRx$Hwu(TXdqO%5nEPwve;%|+fN`$RSy|)c$>Q< zh6E;u1JSzm{f`5~F{MN-ZVuc11-xd(D4OWJkM1=ohplahwFPgeei4LF{U_#3P~C7m z>6a;~hTUAN1$hsF0*2}_KfBWN~j z6S!6lY<#`JRI~Le*f!upB!2ct6pm#cn`XDRbjs$Z=XR@(=92vJ8wrdkO4a5!> zRTw4d7LP?KL41+_X>zyZcF6@c&RAJw`_@sS7Lh>Y&^Tz;p@SU5-RX?<+eDn5SYDe! zI2%IL@!)Y@{j;IPg1q$5+VHDUy5`{yyo7%2ju&}GEA;J~YS~JfcoEHgMk#8M#dG?>`A5tZ&xI`$_kZrlQQO>Yxja{>H zW%-WV)E?YjfT$n(ivH;Q+~eeK$1ccUb~YnB7#r({d6+huZDXdWWdbW%L7L%NE=j0( zHoN6neZC<7533|BJHghp=5271^;y-XR6SMXDLI43#@o5wvK&RubDQjniLGM`#QFia zsQ>l%!9^5u2}6^UR?c^}Y(qO@XkF|r1#^@JdFBAL6YfM0Hp#2L9AlXdNAW-P`lF;` zYuIDBdk)-T2{O`<0kM{)EaZB3T~>_dT1BZV-~a~;Oh}I?APDhghT_2j)JPM(l2sdd zhj`EM&Y-kY%3Kk;l}0kcka64UL*yQaeY}p~H+DMojsJuT%c><=K{W-_dT8l_T**AJ zIGr$v%&3Pihyi|$FkzrM!_1YiM^H4|C9A;ytA`4eSYCB;NZNVM<(BeM*H(%_YYIr@ zT8PIe-Dv=j>9)%x(8tGtdz=FFIKfbo4(|9^u#OV|9StC1 z0?~t^948E2DtzN(05_&XHYTH^C4n`zAdCqV6JQ#9@QW$?U=*P7P~gR6$i?Y?>4d>g zuDlU}R!js{Of@H%A2ONa|)4d`i)#^IQ1r$!Z*tWZ zv7a&&qhxKEzu!w7Hz)&YgsYXTmORfv8^Q(K-cRyQ4bBE zuOykAc}Q0?UQbMon+3TbqA4KxBV6+oGz_%Mz*aW~&9!C>b$8;s zxu}7T*MkuUy6N2^ycAb5?`p*3^F9i-M$YVqrL#+&kLDFbj`oCuCVmihIF%mY6jt@| z`i9*l@u<_+cyq6cN6yw2gR#Nw1=}r@tF~mM5$u-fY0tSbUdP;c8h+F(_gkT8&DDzQ z0sdbvok;S_47f2~})1Ff;ny&lcptXRf!2d{|xnA=bweAze$ zl&sQ|h;Yq0N@xdi<<9u@#sRW#p*Y1FoKe-@h=iSSxNtWV0z-{UPYRE}EXajH+XYWt z5E3df&!cl@dBQl)ze5Bl8%?mqv|LTbFrq7o5ow@;w97A$@8Gtq;Kj^lIax=^l5Hr| zMu$3J{=P6n9$2ROtnmRIK+iGTRCTB6so5ZKW6Z?~?fkyaADua?BR7E*9m+)x9_9mw z_7^j66hbXE_x~a3jBn5K@PYT-9LkA@H2xcu$J%8&xH|uE*7o)sA_FXw7&l8f=Anf< z&ojJm0*c~w0}NzF1iOP%F^GaUGH7i$t~*R;T5h0?Vzy-7IMY*lVy!0k<=K~HPM)6Y zRW5KF!6f>kiHS-Msp0iTD+PK+6d>IL`JV95RqQ+kZv2qkQ|#QF3@CMJ75DF<)IH3Z zEM(3uRx2)ey}ObH&(b2A08Om>oOecVk}H_zwuqW4n4B(A;@9sCh(sMZ2@@2I3K{Et>AJWT-Kq*<1Kew zIhwo6rZ#Vz0j$j09EkcWmg9{>jjeB2a;QCC0fI=5@-Z;6qs@7w#Wk?4lO`3tj!hS&x4L4`|H0*aL_b82KT zu3Ufu&zY!sMhjm`ZLzH)2dFK*u7wj93PiqdDG)}p4Tf0b3;un4osYndCLj{$-{G2_6r36KDR2IpA_cYDom0d+dXz=y`r4J%4g zvAhY?XMje+X@B8c10XKnq5hq8Ch`PpKa6kk&Y{g$z~2z(Yem>Ct9GQ63X=zc#Z#sD zT5LnU{gsVN0}T>-XO*oJqivml6-Rd1^F!JLJR^SCZWUAf$(K)_J$bgL;ExjZ75^*v zueHkCe=k=9c9KA3gXvRerz9HggFb~Xz>faqd281vUq1Qb$+I)v=P%B6?>)=>7xQQ5 zBmf}Tp=f%>RL3@dDGW~Aovd;bYi%ZRlxzm&>a1sXYtLKQe%_9~lfZ7krNC?Q+pVnF zZ^swCqcdtdA2WSNSa2wAX>QGdgTK|m<7mFm+6V4BG-kY_85ippP)|P+C;r~Qm)H{g z2GbT3yUF_v6;gR(@zAPaA4^to0BUuzGFWG}bZ1fh49{&!v(zhAFsktT0g}s#Q7CA1 zM+YLm!||-6F|2#Je@5WNK-H|m&1r%`R+Zc?2zhOco?}NP<~N(CZ`eo@PGqTx(Tv^h z;r_d9rEHxYE=UK+S#fn-J*m`?oLOJ|`D>ZdgF0 z*s3XTv#AB4CI6093J%sAt|hdF=YJmFgpqNPCC1eyjLkJ98~_k4G)fQAHO`9G2y4_w ztO+{=C5&xaN?EXh)jrM@h9ek*`*=zWSZ8N{#fRjpj2a#xC zU{<`azEkjgf`vg#DtP*KUN`Ihsare5;^+-io}VP-2sLqJ{LFIxqoOMpb{*bS}UqhHf7Zm*Q~~cY#_Hf zFKy8-?)YFjCsto&B1hXhZsiUMqk30xDx2s^Eu&ee>j9!`CLk9sca<}n0)h=qab>+r z&1N__=g4cL7cxZ2R)1qk;!txJtj!Ul3u27G>x$yKzudPYJ;=8 z8K|-dW-!2`K#AWh{OYdaHA_g$ zJzLR2VLk4)y7GB9(;99GcWdl;)|YCQ2k%R>0bvog+tMw;({a_VvvoUL)bHBK2|2?% zPZg=8tj`agKGV%tAIr4VMnJUI+`dfd^SBn*jzyr)e=%VVif?&At6~?@V6U+!pa(K> z%PR?1335v-)Tmw2T1Ecl?7NG1|9Erp`qlZ!8H5+Jy={>;{FZyK=}*x3((O&0ZMU8Q z;7zt4n$v9?t!@`P!ajln;P%jc8c{>-DKz-2<}(kcTbg39Nup1Z)ME$^nSEWLFdUPBHB6Qg;{1&W@RyQW9`i>U{+1wL9E+zxCa%+VbVLGI}EN z9E|i%Hsp??w-k8Fg1qPDbMi{41|#6=-fO}%P0LhMno={yB?_}3>AMQYQ;?xh)Hx@mk9yV#TToSId%_2ZA*D$`?c)Wl;p*kH!=& zCg&3I(%G$7kkaDRwX>^_=^({SZp%t;xrZOkZUUm&HR45`+^i#vT_XpN9NNLawaQiQAPzR7&W+yhSBUWP3k;|IhYb= zMv_9!NP;#&_7;bI!h*k4jftvR$tq*U$ceWEM@O-`Hs)&Ie(|$1c#@=<5GO@lzDyq{ z`vijQ@_#mrT)wmn-orW1xw;7slq+@4@gzRG*wUY*Ag}&%ZJKrd3k_P3^TZO!=Wa}c z5_TXPm;TgLOv@!nf{wepFWl(w$xB*OlyQIqh5T&pK-A*P5&&l45l=|Dhdhx1+h4zy z`2?ZDd&sivzv>pg!0*XcQzLJrU}2DocZ}vl@7x)F{J!Ef3v#2RBSzUFwn6oH+SQK(;YtlVLFTQ!V$uldl#Fr$2!=1&wAk|kN7e9wwdEYU=;UWm2(cQ)RZt%(iw6hm>cfFySk&l+`NB$v2y_6$aBprjS>$mN&z`lDoax`5fED$80X#D*_W;m- zPhheSjtWKhlwaa!vE{hI8@_#ad9Sm2*#C*w{#mW>9z3hzhig{d@{C0k2IGQzyyN=t z#gzYA7ZWjY6O&H#_7Qs?I?!<&!{MW@1v`zaz?k>Qh2cd&ThDPQoHUU|0H>MOKl_C3 z9x_DungljhVw2(tW{88A21 z8$E|oY~DP~-urO;0F=&A0x001!n^ku$u{wi6c+x0V zE%4Q!%izWiI%w=W7OXBjG_>#7YG`E5*Us}BW;iH;(Wc&Dt`S&XjW{qk)CkG=%a3g; zGHWPeWO#Hxk66+6dcdgB_L_9kL@^8nHK0fL_lP28T6(G%b#tGwm%%OBQD?LbIz0h4B^x)p= z5C3+j{#nfN?ej8k$N%$j9CwX!VFM9+G^3pL#7#2I#TtvmQe3|Go;UI{NPnND-k)@C z-eWk^WW2-Wkd`l(67tmzdUsS5GgUQ>-2t+>XhpZIW>t$7mPsK@4W+jw6W2A(Zh~Hs zqkQnImk;jY@U_bZQ|L|ZZ5t?t8|atZjlZzp7K004(=yGHWbf|D0`6-B>BC^wQkDSS zJ30giLJePBeHuBpCVY{|M(yOgUE_dlCz9&`!@y&BZWUad$u6|aq;>a2!AYVgG8r^VqLWIN3{MXr;Gr|5F7PJO+`N=2118tq~JGeQe$A+YE4?}a z`|-dQw{h8e2+F|!ZYq=yc+Mk~%DyhgCYAnK)HZ}{B_TlyZQ!cwMqnk7MT za`wU{+}Qx4OHs3GMKgBpTNqfmVOv%O0G(|xq|JHTP>~xL)r?iZNZ7}kReCO7TwjvB z;SqV=u9bF%{KWSQdJ{K{_U|!kug? zI>l<88!|ODP(r28pz}RxtcA?kRkJL(+FVaI z`VT#f!Bstb_yR;($p&KYwxGI~IohZsHzaWJr zFoRn7<~=PkK)AIPGaDeZ?2gWsU%j*C@F?mQA&Yt-Coxp4_Ob_&J+KZB$D5Kzcu|`= zXP-fZ(te9TuFTA;x)YXGPa__3a|HeMlK zoH$|WCL~BeI}a6Wb9pE3Xq8`FjpmPaMY8sOUnsR-6a7710(3m)Fa#W4``)o%(hD|Z zV5_BOWqY2i;F1{ha)r!yL`6=q1DtATDGO3PI@OL&wO@x*?Gl}8`Xq&VtWf*N<2M#? zDr}{j>bH2=ayk@m`)OaEBp#Dwq}jCXEPArNg%JHv`ryFz-x34 zdLGD9M$CG6fv~NeQ*$NIwuNJ-W2a-=wylnB+qR94ZSB~$ZQJ&ax|7?tZr!T$15VX^ zT+eIO8gq>Cfv@D%FHTRVYGZQX!I4cRo6|*ht?1tx$}=&RBmM5xeqcMfb&^twLGLTQ zf1u^JC)Lyz#nKRicVQQ0LQ}WHNUGMNWVky9YKeKYt(9zknh%5;-gd4DoaeHpUX3gc z5>S>5wrN}ihS|lDe!}d?vnrt>*Qrl5WmH7&yJheMuMyi1NB%jO!j|6ZawDA0A#c4B zxQ@i6?qXu_eTIxhuaYK_(wLSa%03i$&IzHOErxh4q|_qgw)nLjk>{Hv- z2&i!*@cd(j_DFi$gPZ~(*kLc+Nx~6^^p;WERKI5>mXg`5PW|WW!NrBKQ*;nv&EWiU z^*bRfS|&o;ly%+f{HV+dk$OuR)t*;p%T4Za+Awe=y&odn!6~|zM~TW>oH)YMl;>Iq zHN}iXRyZlxfjbb|utuuPT733?N?e_Tnr~>k$4p1-#ymdOzgu)|n^|9=tEV7C{g;~D z^2jZU83U(tgDDVrsUUO30RB{|Q8E-%Lmn1}JeLYP$KoVwZ&iB8Pq8!t$k}ryPf3dr zK=E)u!=xP`Ax6v4ea$=`MM{&ZX9q!OrExyh+*r9a6XMwFSxE894=Z%KR{Jlu5=+lq zN%`t7yJ*cgo#%tA>w%&03PoI}0~4L)rll_u>X*wS*Ol*A?coXZ1{LBA-rtLU6@~Vj z0h1}qm^Qn*qTt!L(i)3a;*Uq!TC)QK4nPgJ@t%=3)1NK{v|@GAp3HecrND1rC69jy zStGvH#8vih4Jzn@8z?m4Ehld~G6>l}KH4fO#1i^WSx#d{mRZ7tCUP$w*})oR^PC#E z7E)ZVwEE@&#%++%2>d>zSYSZDqx_1^atc^!+-1u0<0dGL84u<-mL#f{QJ>|&DVu^P z1I58~2`*%q>pk^1daQp2dBa|C7Cl3oFNlO8?l#lG=Yc-+TUIg6*l%=0%k3rjZrMr~ zKa->nZljf0yT{t>=ZfDzJEGIk>@Dq?une@|O{}6r;FPNOs3KC~l|fT|Ls3s+w7=%I zSK^a=?3$x@OYOyB_HW9IkjfEl&Ncc8$ZDMl!BYCSZ$!2=Yj52GZy!#HMSX;wig&bb zVOpGY<3DvIAABqT%wzhkQwYxRiXt&%=KIk>Vu>AJ`RqVVq`l-CCPOpdHB0RRgKXD~ z91K7>0_LgVC$qOXt;5UVAENO#xSusgx0x$0B6GYOg_Fwz7Ib5OuZ7g>foA5G-c4(jTDsC{`*ZKr!!LS_Ji4f zt**-6ma}CzuAJWEI?J1aX?QzBT;tLJQoU$wd7Uv%Zqc$we}mkSWoq%CSw6D$(SJKk zWE#hIH_GA$bo_{!vWx?CHW)8rM59JRo-&_g%z9(U-dhX^JxM!+4`+uDIr zIA4so>FVmy82lrgnLHp4%wnkk{JLx0BQB4GgkXmB%_SrRB%S)&B#f_mLJz{3%y zzlcTM2C?li^dcad38X%eCerVIADd~bn}f-3>iAqEl9n9R1J~X)iS1E*4@5)~g}I&y zVVmLO?%ewxK-m6wc7qVCIYiLk!9Hh_4;d82cn{|2pNoF$m6N59UW=p6l5elrST19J zMJlWYFO$@th5+N-W#sFih>|Y0&zluJ`TfuR zun;yz2qVJwU>Pc|wt6=7keQx9V*xAoU|L_+ZspP)kv?>l??necEHN0=2(o8o=LkbH z%|MWlzvJlZYWlY0S+w$m>p9!FL8J(J{i)~>*3V5j%OXz)=cG?%-vV-?R3*BMQB|xi z)M*&m+)kFZPp{0XI&2lcIih!bt;CrdtEVNQ6Y7D31QYgjoUwY*0F*F8BT2Z;ZgAL& z_4NwkW=ze#ctf0D(du|&4XqhtTp4Eke5FK>HI+g+_0hyXd~*k6lp8CK!O;ZR2ezR+ zL`8_4ltJ5B7=kfGI7`Be>{U9}*dq8JvFbo$wOF^U=S!37JHpMhu!Dwuy1yuV<$pc1 zAmyv^hIOxdJtRU1j$~P7{gO{1Y1&gs$T1_;HyjL>%4pA#uY(;=7S)z!GOC|tl2nqF zo}1vsxv&}uv_(z@RYbJ;{a#1hWXe5ZlQ@cJ&U1y+85H+s#bLKSENxQZYFkfGgWauq!R-vFS7xB`nd; z#69yoNwLnC;`rJFwPIz-NQ&lL#1wVpdaG>0C3hsc!gx{x{&c0SY_%g2;-G^np5PvZ zL;5NBZ=gB?n6BBq<;J<)Xuh@?BCCI{`16e#$@36GOaKgITcSjrX2mGq@ua)KQaWWb z+3-4(AqzBr9W;Np+s#)QgGJJwfABn=64q700MW&lCM;Bq*rl=3LChJkQqOyF&bVwp z%uqHFTJL(=xl=()Uu~`E*$))$K9p6n zyV}GF8TT+BI*?4`NNmYJ3-n9mEo=^rM0wS8VM2B<)ihztJ;+c{epWL7M3#nsFmzJ% zXkQqaVeh_v9`t#O=(;J+9j>L%FoKpJAgveig`a%;3OzS830>x|EQb>`9p&G4mY zYygy~Xtw$G(aWl0N3gw1YA)+^4 za$kE#tK5gGRFExFejM8NeDZ4bHW3b^U%T()Ivxkr)bTJWgR=`^e+x`Ff|>+jC-s5^qzu;2_<_Ib=T3_1k}m>6QJrW(>af%B-LJKOByog^eoGkMZC^@9wbj(|d*lkSQSw5kYUj&pR3@g1NbSs3z6! zPsU?Lo+$D=H}g{K_mt5MG*l`*;;$K4Bk@E4K?8_ST4({Q!=K@=fT-7&{16I)`}zsd zmqo3uc5DRiM^K8plGY$NBCTJ^=HTQy+PILr?_!)T1I;y(C%#~JJ05+2b19U^daV{Z zF3)({_Gm3M?;3LbO8IW6?tmiO(7<{mMaw2iW6|Pbl zQQ5h@VD!C?*D|Q$h&WX7>En$i8duq@)V3rG3k`wlZTsIO5IY&ONL;(6>WkFAxci#Z zWoG=8Ei)TWCbV&u==w>7lWO~rSI1QRGeB3fW!~&x?GxW0`xM#wgOw~u>ZdeHGQueJ zm60cQjOV-=z74Gue74A5{8ljw$cM_exwE4)AA#L27NlIXU{04mhacU8H)xAA$ydGU zBq>wuYgrvk76B|o0^|vRIxLPUwEm!Arf9tTcQT^{p>hz2e%T9?U=Hz5hpBT=11(_d1VdHK3TtaC5B?TY;9;@m(&DpV?EO^ub8QhJV z)PDx-@iffFve4^Amwy;(@myH`wtXOC91@lrf`8L7eK$5{ZRsR&%(C>6>#^EHR<$Yj z(^GOSFWLI&LiY}AEIPg5C|Kf}vOT>{L*a=&&I`Au>Oe^kO&vz=OeFD<`%+L59<_G` zgAeVh$(TvIL`Fo-BuyyQYqCQ=MGmNz=7=t$rp;#R!N@~+GeDG?;vz6u%3_N97dkYK zw7(-GlVYL`Y<4&D$@=Z4CS&H5-)TV+@8WE*uwEKryxJ&8lqh1olvYfo-)z&Y1&xND z*Vt>eA&8>9tVcy%Wk+xY)fF9eLP?43VPbdLz`IhR!V?R+)X0x46+tN3>MqX=aq#1Q z@vmO8x*UbAQi2V!jnQjQ^<&F-Y}P_o7Fe$jkc}-W$SGGh%c~IsJjWdQjWHci<#>ac z?LUk0lBvt0YxTB>Q>zz%U?Jn46E|Oa9DdNCmSyc8?io2XI8Y{LAYzmGR2oOrjhItt zp(!`G8Uho;B$?#4I5TBU}5j?=9@+qXAN* zX3bg@7;EEO-u*~_|Eq({;n0u_JlZ!-4lM<$ghJO;JlEt8a--5vwej1ZQ0iQ=IJP<- zocWWkxA`JXsJ$^y;)cPy4Z7M{k;QXT@;hvDAE;{9d!gYr2|}$~I^zW>)0xfFF-~7) z=0|VwFhg;rNE|&A-cPxfvJJR9Fd2kBnhXJDQ}R>_o2;7ldvcaA#&Z5-LFLKZBAM6? z9+WKLu<_F7%|>9?VS@va&0c(}7bsWD?I+@8o*6h3d&TVF3B=qFhs_Gc7LCWK<77T&7%>)S8dJ;)?BN{4=fvOX6K+})>>kRG_KMeffo&Dg z#7Ud9y32!f4k2t$r|b*Oofez(#6|jy>xRlpL&mY>q8@pk*R!isW#A5|kGsmD2xN)G z>5Y@Rk>w$9;`>}pk9(V44Vc=9`kPJ#d0c<@{9YLdY;Q`uEY_?ACaT8sJq7fS<&%Di zo}bK1k;rvPOVtmJPL)&UQOZH_?dCz0e`HA$nPbAtWzL^EY79AVQY-mL*K(yDgKvV& zHR?MqEPL2u8cxHT&Ze?H)0Q<@Vx#*ddW)kQtc8^YPiSLg58OO6V;TK%reBBVgKi?%@aql(&(oA~e-ftsE!~3VoqrLLndelZkd589CN9TNJ*yg56015 z_v`m}vQ+DM(0Hc#H=1nM3F_l;bQoPVIJ|V&!0FAc<rDc1cf(>-_%jVP_FZ&F~t2S@VEq<2*}ct;tEn*bqG2(>{i7 z%wD_GcH<4Hm!pau2cPE|&b9Q)CDUD}V4^<(K2;=q{`;o~g3E<)P~=~Kk2SSFjtFk>HfS28HJ9VS{tgdduU6i0w!^^%pCqy@*ci{sG;v2GE91G0!M_j`4kh*V;?VJN@Ugr=vnLhv4P`COrg`puTV7}Gp9d2L2vLBtPY6@@uaq}?J`8+ zr1c&zq_kCyb*!pBUmOdk!la=&?VN#V4N1zgHD>H{ynlMfqKK-eaR0t9gE%A0nI6|< zEk^(T__Fe^icAg1hn`e}^@!64H}5S24bRQ>wH|r+{AtKTNOElt$n6^qi`PmSYeUgq zOfjXZ+h=hy>o^oo7KY?;-KU#PSaJwrCc|DhF4x#uAnH%r{53mqvNH2oy@utXk)EHz zc!T9h#Wy#RYdbvx4ws<#-V{jE>tHb+3X6dlPeO#GBwP!*MsVV-37mqXwtb5%w60ls z#UJ#`s3&9&u=g_yq$1^nxxDP^U7ANOmFI?8_9P!LneQAFJNLErAEUKeR-!9uTSC?% z*31jb%AZp_O-ylO5p)XHJP0O-j@s<1ypW6BTi*3_5$ScGs2M`V+lyX%HcCSbHJig4 z^@LxqxfEa5!fW-KBebI*>#uA#^!;A>`rE&hF%J*_VD_0f@q)Wxsos06 zr5hwW#|^|A;|^uygbiapl|g*J6YymQW3-e`+Yc9dYaLYG|{% zW|zInr&kTEbcedrD_=s)5ixxkhHzOnSI6cZTKl_pLNGE$od7FM^T($OGirFRy@Y`r zCX~K@!(*J>Oq^YVnuTigIElLLP&U%1#cq9Z3pC%yAVN*5>)Fwa0F!}xLr{SNeMl-@ z^6FK5mjLa>!}|EV*93fEfOna!egHle^(9+5i~oU+EeYj-6CTt4go}8tBdqdy>Wf1qgX^Oj=)ZKhcD|zLodVAXOmz%4tt5yS3 z<3FxqATI^M-BXN>S1e%*_!8doC1FPF)*RD;-z1-3mn$A&~I^)C|0_7vWZ#T;z< z-|vUv^PAIotxRz~GEg(v4$NcRUZKDq{?Rikd=u9Q#C(VM1_`=5RmFP~#HD6_S)d8x zXPtlu)mI)TV$S(DLoY2u`>E;_&4k0e_f)o$b?FJNB6bGzyX8)_hv9UWzCD`lqlhqc zFn1rO1T7Bj_I7-L0BJD)hJe$nzgCu$f6ghm-81 zDBsqRU}StLdc=nVJH6w_?$5#LwY;yl-4=b9pLS^nxozpo)?}M8aJ(#f%7xRNEq}== zS0}tg6x`H*e7ye11OK&5g04}Y@6nEL@Q$BR$nbL=7N(15x5zC2^$HF64fQ3_dk81u zGJ1c0U{p&-M<->^DRFUp(znD{CosnAX+>UX&1SSM%&c_KvtI7~OB+z!wZNPbrCiSdbHy=WMz!8WdtH!1<>Ak?vUFB3Sg`;sCMJJ`p0 za^G-s04WLZt`@fvnZ+Ht@5!ZJcg>wME`q)VeVdctboz&&Dha~-eu^!e30_}bPV2h! zeq$;;s)`QG57-d4GpUZ0u?HDor-U->?RWH8K|X~q(1T-2_cW@!|85_9`_*;j z8^@3x>9!da-=&b~;b5+yJqUPX?uuRLqZ;I@gp054p7rkw*U1P#TY`&HEN6*>~(5ue%Nk$)l}2y8zBC zavm<~9`8LP*IuW*gV$-(+4^MdXxL14KgQNvAm{z?)kCn|Onm>F*_{knj*$wsN3zmi7JEmmD=}9-a?$3DqeZ4upOlGC| zrl~3=#eZcegGaw=vc)=e@ZOzrdvv&bes=fhSvYEv0A*|rQSgBwo_#uJFkkKQyU#k# zB{U&@9MF=il{7=UAJF#i5P3)X_2CFjj)-H0hRtxZiwzx-S2LyxDiZ-hmGfK_hj|i5 z{K2XiVOki}BG_~~nnoV&{32{MR#8m%Wa)~BQR?p?ny#u@;P%4}jTtv$5vM<4YQghs z-^YS&bUruKBY(Kw7->XV<|K#5VW%1Op{|cWBxRO`g|wP}L^T*$J+4%1So-+BbnX3( z2NBYoW$C>|C4O7}{bXZ_WxU3nPUCTBekwx z#OX7vc$T9Jb>LB0lW@60HHi9KR-Ykcu~nCQb{{VO{xcPvh|>qKPnvo>kQ3}~r11nb zM0ddZMG-`tlN2n-)Tj!=FOVa9XicRdhm*9YUN?%w-X+^GKq405CLv+C*~%{lWa4Ea zjj25Wn@GZn#Se#}G`Py<%%folp(k2mBG8s{LaOFCBup}@N15`&yeq?hSRXc#FvHIE z(^!!SP>$)2q{83Ay@^@VL1$`REBojEysG#ZGLP%c7d6w}ng8q!YTo(Mgiy2L;XI3i z6S1MUPzv9L)R~*ug9z%Jv`~8GDogB0Eol8j1Ndvoa+j^%?1>@q@A-z5$~)z{U@jHjfc8A3A}k zH*>aQ78g_4CvP-n^Eh9UpresZn}hYT`*DA%3VY&VVcTHm(WM1 zB{aV)oHPb+4`%P(zTa820u_eU<{qv^EFgT0AXioAIFx^{ zhWyZqlr96>0^X0`UEh_RUl0&VGq^t+S2Dh{@Z@AIHx~z0ZljH!jq_=^&Kh9KoE~PA zv1t8(oFOnIuct0&vXh6!^@Et0^azpvyl%RI@UCq9;n*6<9Hr4ew99JP z^ZEUlsnJ`xjy+b%v9#HxHj5w0zhu!LK@xeJG3)HXF25|@=so;*pDsyG!go^TFLwf{xuU&F9XE! zQ)%7IxNEU8IT)6=xoLVW@Z!^cMG<0#RO_OlE;ec-$Cg@nZwpK!IFVzIc`<1ig*?4_ zH;T3JXjOt!=hoU3`NT|YEI&$G*6XyO$4y*J%YD7S!eIRU1M1wpkH8JODC7@4J(`}( zR}b=7(0mx6Wh#E8Pf`v`RcLL|Yww3`j)Ic>W+edGdp_Qen>5y(mNa-qXgq5Bm&Cqn z@<}NU;5F6lE;m$*bif=Di~_JXGNwt5XNoS1Vk(Cs2EvP%!*6qIfgJ$AZ;MmXb2RJx z?B}rdL2r1C!m~-?v5${He5nw>kNf=&p-V4ypfQc9Q8rtPMOInoi8G&p(^2~03qK2 zfI>6b1HuFnR6|-UWFK%v&XlrC(k{(5l)=N;PVG7ht*mY3-#}4hLuO4Ib*&VtK^aXO zbt^Qifq>m^(?D#fGZYO6d)4a6kD$ZL!&bqo-NM#Hmf%=VRuGp4`jnH>1{{>#=35@` zm%(fSk}FZ}<*XYT&IU%N0)9@Ma>x_3_Mjdy8hu)SKqgr`DK@l9R*9}tXrr3cT%pYb z7K@XSNWLb?${WhkbD>Ccl+i)9CRkXz*72;P8JO3SPe}eyQXjszRPxVhk2|?KWcX%l z(pVhb8^gT~zGAemMXP}w$mYF<9Y7P*iWy|rMh^y|aNa<;+CqPz)Gp4i<+F3UiR+-r z=88YJA=Oo)x0@7kdFR+Kcu*kD7DR|WjfBIQC;XRyv9r7V+THPAt%_dFc+)pYsqL2g z;^sPn!4R=V2EH%x0ZoYw;kx)OO8wJM}D%1!#llP*OgUUV~M)u6T2Q@*9z) zbWDM2KhiX$NJu%@f)t9wua~sFl;mUM8%%;w>eVoie$MQIxsXzPR(+$WwRX;kCTn_i zOkgvQn_Epx&W7fKSjjMT^Lttdr zrNl!G;AW>_`+VL=6R+M>5Ydtp?C7m_0$O^ilhYR)c_R7_6wfl((vzie|6VpdbdS2z zR;~h2^HaV(KH3cax&R*)%gY1&*gx0$zAv90T;B2xdjBj<1#|}dxjdO#TI%~cxk>ln z)Cf3Z{C?L^t|PoR_~!dd+x6i?mXyx;{Va&aySWJ8KF_K(<0vf{0W(hKY~T6HyEe13 z*WXk^4FmBTV(3GuhH)G~GQT{f5w`%0z`A zz%}he5v4kue4DgV0J`dLDENxwzwHWWq@e7Djo5G8(R^{#xHuJXERW~afOr)=hdE{I z{~Kzy{Nm-j{~zN`m*4+8phe3{R##DKG59$nXVzdVU%Vrg)J+2{?n!2im>&uP^^qXI zJ_m)~H0rh9OMd82bqzL1i1rk&C3BzP&YN{n$B zJLpq{9z3?)TJN3c`cAYPjH6UgFyuFmX_TUCoZBVSmGQyiABQnWt;w%QC^0@GUL?m@ zeDS~a=Cjq8k>wf)MYJWYCg_3F%x~zQN{?_HpY&5D0$Y~1H#=N!`JjU-jbGv-sO>WA&Ir*CqvV(%dbJOh+N>v!gDg(%7TA0b?Yj< zqZF`3nv!(Jjf*pV8j8eO;f1fKdb5AA3Xc@rHOCdfC!oEPAy>sE`FxFwS!g7~q%2^u z`52N=?on}aze~)wK=#tpzK_qGntDY5rAp|s${HA&XmjI@S9)31sv-KNp6;=~l`TK& z-b_Y~O_fDm_JsLZQe4=g>zsd`2aevJU;7#+ z=Em*(a{Sf&D($i3p$*IZQr zTpcwcM6&k!_=oIoLQq5p1IrC-W`IR8tQ&d06vG5sC_W$V4r2Gmq}H$C5nXj>&nngq z2HK9(Nj+N3f2b)0h2FAP*ew)-R|#mY?6;H}RZNQm1s=OT&582Nv?lEa9Ku+?7~FgC zEg_W6mJFYmJBZ(D9##mh0dkTrZJ zf#|f96C?d`2ul@94+Yz2%Pf^u_b%oCoDeJ?Uu{aHar39TOSRnEl%^Q6>x)N*wrQ>v zqe6 zFwl6V-V8|VHTM%aV?%(e;WyH1v0qw<|D$uF%1CNKjPd%?=I;C##$#E*Yv_RBtV`{5 zz{Jgs#=dlA*a{?+1*k84+V@ASiYeS@zM844bpPG}Z)nUmXfV{v(KTA}z~yrJo?-n$ za8eh>9DzO9SrlN}f#i*PGcx5~e_5bus8gC5AL6bZZwl7D%{^`3L6ti5L7B*K9p3Q7 zy0!tow&AZN^q|6@ex>-!jVB#Xi}9{r0E3ufg5F!-*coy6-r0kvFVuuJ~7P z*=~%W4%B8^Sfav~18jFU0sVJ(N8xbJwuf%_Ow8qiU9?q?YKk1`TPp>^ICc~M)&yj> zh>Nk*t}V2}Hi?eq^((~DYT_8>jnZljL6mj=pV@oU6XbOuhLJuHH=}04v`~Xo=G;LF zJhPYpWI?6WD#~o@ZKRD#kroHw@~~~?`gKJ+$dK}#h$7U-iy>6!6g^?)Tt)ZOu8){$il`1^#~iZOLE&&EGZSxlqA6%OEU2# zJg_9^&{6^`U56=;9Y4v-vEDSeU2bXk&mx73t7fNkgrF3286^&jLY9a33~R0Za)qbS z5p#C(r8X{sr|C_`@XeAF9+^^I@gBoxDXX_$jDymhLc-Dah-~+1=r3mMKQaw2?LyMA+?eQ^OL!)QU`uLRqnP zlN)z%$mjL&=LGM=k3(e56&o7!Q-}~e#&$Jmivpjh=_vkc&z9g5B$Z9;fq)7vxg@sD z?Mdk8RFEFnr;tVAvt^6M*sPyE4cffH`LrJW;18s1x3Nw?(3>5Ie3}a9Yt_S|afRBe z7wp5aI*PN54yM_I3A!EGG*U#v3CXJZ#;hk~~c}Hrh^j@953x^?fVaPSIhvNFy zIsW6^;ruR{hE1qub-U>R{cCztlE}B1o?BpdFu!avu5}XB+_BR%S&^9c$5fmN=4O7& z_PO6ToNHQuuD3oD4LJRODk)6`5Nit``EiHj5Wv1KwH9{KeJL6@6=VLUV0fc z#T$#%y?Lvzd#aV^X(#qvdaAIVu7r`S^&#lvjkdB zwQql8)iVo6lWz1{ghwsr zU6ruwd8jM}M|$@dVhm5Tze-cPgjF$|etcT9+AvIC+uFL7r2Ek-RQsIXLBkwboAX(i z5p$l>Ipl8{g*+#MZa#+%UB|_Fo-oT9>Ui$l6d9)KgBMp`k8a)WE41lc;nnF`h zmUSyJ2>G2A`~~tWb4hQ`oz$Wv#1A|5F=p03%qoX`cwe>LGUU}|mT|AR`rqYm^yDxc z>>wraDeK@4ZGV<9#C1MP;Qp1iqG542FKNH~?0x(@JLiZ&gysjlVr{Hvg4>&Of}X+q z@vzo6*LG%}_}rK0_HXr)#IBmof;YR;!Yrrb!Eqm`i*XOe`P^{Bui9v{EvUg%B*=!S60xi<|3(0EHWsLb{t<4y2R%z$I`H4Mi zbeBJUhD08hRJ%~SFB5Wd8)WC5<~X+2CcO{6ro(hXUZb|R>9bh|V3dFmpwmQsAG%nL> zIR$GLNL=U~RG&Nl7-4j-(umc-1Y`m#X90q^$cd&ekBMN~^uSbkaJavuNMjqcML$LF z#jGguxyptZsJ2^ZKhEVMChxJJEfK=khT^*O{bpL(6KD4xOg;PoANe;MtUHvOO@g%| z_8vY^Xitp1dm_TbAnZ-6G56`-2Vq{4>$~>Hg5OsLUt=yl_H#EKnZR#w5RmV00UqKe J5EvSe{{b!B%sc=9 literal 0 HcmV?d00001 diff --git a/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/_helpers.tpl b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/_helpers.tpl new file mode 100644 index 0000000000..146bc45a14 --- /dev/null +++ b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/_helpers.tpl @@ -0,0 +1,30 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + diff --git a/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/jobs.yaml b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/jobs.yaml new file mode 100644 index 0000000000..6955e3b309 --- /dev/null +++ b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/jobs.yaml @@ -0,0 +1,102 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Chart.Name }}-create + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }} + annotations: + "helm.sh/hook": post-install, post-upgrade, post-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed +spec: + template: + metadata: + name: {{ .Chart.Name }}-create + labels: + app: {{ .Chart.Name }} + spec: + serviceAccountName: {{ .Chart.Name }}-manager + securityContext: + runAsNonRoot: false + runAsUser: 0 + containers: + - name: create-crds + image: {{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - > + echo "Applying CRDs..."; + mkdir -p /etc/crd; + base64 -d /etc/config/crd-manifest.tgz.b64 | tar -xzv -C /etc/crd; + kubectl replace -Rf /etc/crd || kubectl create -Rf /etc/crd; + echo "Done!" + volumeMounts: + - name: crd-manifest + readOnly: true + mountPath: /etc/config + restartPolicy: OnFailure + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.nodeSelector }} + {{- toYaml .Values.nodeSelector | nindent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.tolerations }} + {{- toYaml .Values.tolerations | nindent 8 }} + {{- end }} + volumes: + - name: crd-manifest + configMap: + name: {{ .Chart.Name }}-manifest +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Chart.Name }}-delete + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }} + annotations: + "helm.sh/hook": pre-delete + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed +spec: + template: + metadata: + name: {{ .Chart.Name }}-delete + labels: + app: {{ .Chart.Name }} + spec: + serviceAccountName: {{ .Chart.Name }}-manager + securityContext: + runAsNonRoot: false + runAsUser: 0 + containers: + - name: delete-crds + image: {{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - > + echo "Deleting CRDs..."; + mkdir -p /etc/crd; + base64 -d /etc/config/crd-manifest.tgz.b64 | tar -xzv -C /etc/crd; + kubectl delete --ignore-not-found=true -Rf /etc/crd; + volumeMounts: + - name: crd-manifest + readOnly: true + mountPath: /etc/config + restartPolicy: OnFailure + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.nodeSelector }} + {{- toYaml .Values.nodeSelector | nindent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.tolerations }} + {{- toYaml .Values.tolerations | nindent 8 }} + {{- end }} + volumes: + - name: crd-manifest + configMap: + name: {{ .Chart.Name }}-manifest diff --git a/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/manifest.yaml b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/manifest.yaml new file mode 100644 index 0000000000..8dc9dfb447 --- /dev/null +++ b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/manifest.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Chart.Name }}-manifest + namespace: {{ .Release.Namespace }} +data: + crd-manifest.tgz.b64: + {{- .Files.Get "files/crd-manifest.tgz" | b64enc | indent 4 }} diff --git a/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/rbac.yaml b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/rbac.yaml new file mode 100644 index 0000000000..a4d498b0fa --- /dev/null +++ b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/rbac.yaml @@ -0,0 +1,76 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Chart.Name }}-manager + labels: + app: {{ .Chart.Name }}-manager +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: ['create', 'get', 'patch', 'delete', 'update', 'list'] +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ .Chart.Name }}-manager +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Chart.Name }}-manager + labels: + app: {{ .Chart.Name }}-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Chart.Name }}-manager +subjects: +- kind: ServiceAccount + name: {{ .Chart.Name }}-manager + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Chart.Name }}-manager + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-manager +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ .Chart.Name }}-manager + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-manager +spec: + privileged: false + allowPrivilegeEscalation: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'configMap' + - 'secret' +{{- end }} diff --git a/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/validate-psp-install.yaml b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/values.yaml b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/values.yaml new file mode 100644 index 0000000000..99e63600c4 --- /dev/null +++ b/charts/rancher-monitoring-crd/104.1.2-rc.1+up57.0.3/values.yaml @@ -0,0 +1,17 @@ +# Default values for rancher-monitoring-crd. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + +image: + repository: rancher/shell + tag: v0.2.1 + +nodeSelector: {} + +tolerations: [] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/.editorconfig b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/.editorconfig new file mode 100644 index 0000000000..f5ee2f4610 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/.editorconfig @@ -0,0 +1,5 @@ +root = true + +[files/dashboards/*.json] +indent_size = 2 +indent_style = space \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/.helmignore new file mode 100644 index 0000000000..9bdbec92b4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/.helmignore @@ -0,0 +1,29 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +# helm/charts +OWNERS +hack/ +ci/ +kube-prometheus-*.tgz + +unittests/ +files/dashboards/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/CHANGELOG.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/CHANGELOG.md new file mode 100644 index 0000000000..8178169b91 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/CHANGELOG.md @@ -0,0 +1,47 @@ +# Changelog +All notable changes from the upstream Prometheus Operator chart will be added to this file. + +## [Package Version 00] - 2020-07-19 +### Added +- Added [Prometheus Adapter](https://github.com/helm/charts/tree/master/stable/prometheus-adapter) as a dependency to the upstream Prometheus Operator chart to allow users to expose custom metrics from the default Prometheus instance deployed by this chart +- Remove `prometheus-operator/cleanup-crds.yaml` and `prometheus-operator/crds.yaml` from the Prometheus Operator upstream chart in favor of just using the CRD directory to install the CRDs. +- Added support for `rkeControllerManager`, `rkeScheduler`, `rkeProxy`, and `rkeEtcd` PushProx exporters for monitoring k8s components within RKE clusters +- Added support for a `k3sServer` PushProx exporter that monitors k3s server components (`kubeControllerManager`, `kubeScheduler`, and `kubeProxy`) within k3s clusters +- Added support for `kubeAdmControllerManager`, `kubeAdmScheduler`, `kubeAdmProxy`, and `kubeAdmEtcd` PushProx exporters for monitoring k8s components within kubeAdm clusters +- Added support for `rke2ControllerManager`, `rke2Scheduler`, `rke2Proxy`, and `rke2Etcd` PushProx exporters for monitoring k8s components within rke2 clusters +- Exposed `prometheus.prometheusSpec.ignoreNamespaceSelectors` on values.yaml and set it to `false` by default. This value instructs the default Prometheus server deployed with this chart to ignore the `namespaceSelector` field within any created ServiceMonitor or PodMonitor CRs that it selects. This prevents ServiceMonitors and PodMonitors from configuring the Prometheus scrape configuration to monitor resources outside the namespace that they are deployed in; if a user needs to have one ServiceMonitor / PodMonitor monitor resources within several namespaces (such as the resources that are used to monitor Istio in a default installation), they should not enable this option since it would require them to create one ServiceMonitor / PodMonitor CR per namespace that they would like to monitor. Relevant fields were also updated in the default README.md. +- Added `grafana.sidecar.dashboards.searchNamespace` to `values.yaml` with a default value of `cattle-dashboards`. The namespace provided should contain all ConfigMaps with the label `grafana_dashboard` and will be searched by the Grafana Dashboards sidecar for updates. The namespace specified is also created along with this deployment. All default dashboard ConfigMaps have been relocated from the deployment namespace to the namespace specified +- Added `monitoring-admin`, `monitoring-edit`, and `monitoring-view` default `ClusterRoles` to allow admins to assign roles to users to interact with Prometheus Operator CRs. These can be enabled by setting `.Values.global.rbac.userRoles.create` (default: `true`). In a typical RBAC setup, you might want to use a `ClusterRoleBinding` to bind these roles to a Subject to allow them to set up or view `ServiceMonitors` / `PodMonitors` / `PrometheusRules` and view `Prometheus` or `Alertmanager` CRs across the cluster. If `.Values.global.rbac.userRoles.aggregateRolesForRBAC` is enabled, these ClusterRoles will aggregate into the respective default ClusterRoles provided by Kubernetes +- Added `monitoring-config-admin`, `monitoring-config-edit` and `monitoring-config-view` default `Roles` to allow admins to assign roles to users to be able to edit / view `Secrets` and `ConfigMaps` within the `cattle-monitoring-system` namespace. These can be enabled by setting `.Values.global.rbac.userRoles.create` (default: `true`). In a typical RBAC setup, you might want to use a `RoleBinding` to bind these roles to a Subject within the `cattle-monitoring-system` namespace to allow them to modify Secrets / ConfigMaps tied to the deployment, such as your Alertmanager Config Secret. +- Added `monitoring-dashboard-admin`, `monitoring-dashboard-edit` and `monitoring-dashboard-view` default `Roles` to allow admins to assign roles to users to be able to edit / view `ConfigMaps` within the `cattle-dashboards` namespace. These can be enabled by setting `.Values.global.rbac.userRoles.create` (default: `true`) and deploying Grafana as part of this chart. In a typical RBAC setup, you might want to use a `RoleBinding` to bind these roles to a Subject within the `cattle-dashboards` namespace to allow them to create / modify ConfigMaps that contain the JSON used to persist Grafana Dashboards on the cluster. +- Added default resource limits for `Prometheus Operator`, `Prometheus`, `AlertManager`, `Grafana`, `kube-state-metrics`, `node-exporter` +- Added a default template `rancher_defaults.tmpl` to AlertManager that Rancher will offer to users in order to help configure the way alerts are rendered on a notifier. Also updated the default template deployed with this chart to reference that template and added an example of a Slack config using this template as a comment in the `values.yaml`. +- Added support for private registries via introducing a new field for `global.cattle.systemDefaultRegistry` that, if supplied, will automatically be prepended onto every image used by the chart. +- Added a default `nginx` proxy container deployed with Grafana whose config is set in the `ConfigMap` located in `charts/grafana/templates/nginx-config.yaml`. The purpose of this container is to make it possible to view Grafana's UI through a proxy that has a subpath (e.g. Rancher's proxy). This proxy container is set to listen on port `8080` (with a `portName` of `nginx-http` instead of the default `service`), which is also where the Grafana service will now point to, and will forward all requests to the Grafana container listening on the default port `3000`. +- Added a default `nginx` proxy container deployed with Prometheus whose config is set in the `ConfigMap` located in `templates/prometheus/nginx-config.yaml`. The purpose of this container is to make it possible to view Prometheus's UI through a proxy that has a subpath (e.g. Rancher's proxy). This proxy container is set to listen on port `8081` (with a `portName` of `nginx-http` instead of the default `web`), which is also where the Prometheus service will now point to, and will forward all requests to the Prometheus container listening on the default port `9090`. +- Added support for passing CIS Scans in a hardened cluster by introducing a Job that patches the default service account within the `cattle-monitoring-system` and `cattle-dashboards` namespaces on install or upgrade and adding a default allow all `NetworkPolicy` to the `cattle-monitoring-system` and `cattle-dashboards` namespaces. +### Modified +- Updated the chart name from `prometheus-operator` to `rancher-monitoring` and added the `io.rancher.certified: rancher` annotation to `Chart.yaml` +- Modified the default `node-exporter` port from `9100` to `9796` +- Modified the default `nameOverride` to `rancher-monitoring`. This change is necessary as the Prometheus Adapter's default URL (`http://{{ .Values.nameOverride }}-prometheus.{{ .Values.namespaceOverride }}.svc`) is based off of the value used here; if modified, the default Adapter URL must also be modified +- Modified the default `namespaceOverride` to `cattle-monitoring-system`. This change is necessary as the Prometheus Adapter's default URL (`http://{{ .Values.nameOverride }}-prometheus.{{ .Values.namespaceOverride }}.svc`) is based off of the value used here; if modified, the default Adapter URL must also be modified +- Configured some default values for `grafana.service` values and exposed them in the default README.md +- The default namespaces the following ServiceMonitors were changed from the deployment namespace to allow them to continue to monitor metrics when `prometheus.prometheusSpec.ignoreNamespaceSelectors` is enabled: + - `core-dns`: `kube-system` + - `api-server`: `default` + - `kube-controller-manager`: `kube-system` + - `kubelet`: `{{ .Values.kubelet.namespace }}` +- Disabled the following deployments by default (can be enabled if required): + - `AlertManager` + - `kube-controller-manager` metrics exporter + - `kube-etcd` metrics exporter + - `kube-scheduler` metrics exporter + - `kube-proxy` metrics exporter +- Updated default Grafana `deploymentStrategy` to `Recreate` to prevent deployments from being stuck on upgrade if a PV is attached to Grafana +- Modified the default `SelectorNilUsesHelmValues` to default to `false`. As a result, we look for all CRs with any labels in all namespaces by default rather than just the ones tagged with the label `release: rancher-monitoring`. +- Modified the default images used by the `rancher-monitoring` chart to point to Rancher mirrors of the original images from upstream. +- Modified the behavior of the chart to create the Alertmanager Config Secret via a pre-install hook instead of using the normal Helm lifecycle to manage the secret. The benefit of this approach is that all changes to the Config Secret done on a live cluster will never get overridden on a `helm upgrade` since the secret only gets created on a `helm install`. If you would like the secret to be cleaned up on an `helm uninstall`, enable `alertmanager.cleanupOnUninstall`; however, this is disabled by default to prevent the loss of alerting configuration on an uninstall. This secret will never be modified on a `helm upgrade`. +- Modified the default `securityContext` for `Pod` templates across the chart to `{"runAsNonRoot": "true", "runAsUser": "1000"}` and replaced `grafana.rbac.pspUseAppArmor` in favor of `grafana.rbac.pspAnnotations={}` in order to make it possible to deploy this chart on a hardened cluster which does not support Seccomp or AppArmor annotations in PSPs. Users can always choose to specify the annotations they want to use for the PSP directly as part of the values provided. +- Modified `.Values.prometheus.prometheusSpec.containers` to take in a string representing a template that should be rendered by Helm (via `tpl`) instead of allowing a user to provide YAML directly. +- Modified the default Grafana configuration to auto assign users who access Grafana to the Viewer role and enable anonymous access to Grafana dashboards by default. This default works well for a Rancher user who is accessing Grafana via the `kubectl proxy` on the Rancher Dashboard UI since anonymous users who enter via the proxy are authenticated by the k8s API Server, but you can / should modify this behavior if you plan on exposing Grafana in a way that does not require authentication (e.g. as a `NodePort` service). +- Modified the default Grafana configuration to add a default dashboard for Rancher on the Grafana home page. \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/CONTRIBUTING.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/CONTRIBUTING.md new file mode 100644 index 0000000000..f6ce2a3235 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/CONTRIBUTING.md @@ -0,0 +1,12 @@ +# Contributing Guidelines + +## How to contribute to this chart + +1. Fork this repository, develop and test your Chart. +1. Bump the chart version for every change. +1. Ensure PR title has the prefix `[kube-prometheus-stack]` +1. When making changes to rules or dashboards, see the README.md section on how to sync data from upstream repositories +1. Check the `hack/minikube` folder has scripts to set up minikube and components of this chart that will allow all components to be scraped. You can use this configuration when validating your changes. +1. Check for changes of RBAC rules. +1. Check for changes in CRD specs. +1. PR must pass the linter (`helm lint`) diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/Chart.yaml new file mode 100644 index 0000000000..2aa440a1ca --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/Chart.yaml @@ -0,0 +1,126 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/prometheus-community/helm-charts + - name: Upstream Project + url: https://github.com/prometheus-operator/kube-prometheus + artifacthub.io/operator: "true" + catalog.cattle.io/auto-install: rancher-monitoring-crd=match + catalog.cattle.io/certified: rancher + catalog.cattle.io/deploys-on-os: windows + catalog.cattle.io/display-name: Monitoring + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/namespace: cattle-monitoring-system + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: monitoring.coreos.com.prometheus/v1 + catalog.cattle.io/rancher-version: '>= 2.9.0-0 < 2.10.0-0' + catalog.cattle.io/release-name: rancher-monitoring + catalog.cattle.io/requests-cpu: 4500m + catalog.cattle.io/requests-memory: 4000Mi + catalog.cattle.io/type: cluster-tool + catalog.cattle.io/ui-component: monitoring + catalog.cattle.io/upstream-version: 57.0.3 +apiVersion: v2 +appVersion: v0.72.0 +dependencies: +- condition: grafana.enabled + name: grafana + repository: file://./charts/grafana +- condition: hardenedKubelet.enabled + name: hardenedKubelet + repository: file://./charts/hardenedKubelet +- condition: hardenedNodeExporter.enabled + name: hardenedNodeExporter + repository: file://./charts/hardenedNodeExporter +- condition: k3sServer.enabled + name: k3sServer + repository: file://./charts/k3sServer +- condition: kubeStateMetrics.enabled + name: kube-state-metrics + repository: file://./charts/kube-state-metrics +- condition: kubeAdmControllerManager.enabled + name: kubeAdmControllerManager + repository: file://./charts/kubeAdmControllerManager +- condition: kubeAdmEtcd.enabled + name: kubeAdmEtcd + repository: file://./charts/kubeAdmEtcd +- condition: kubeAdmProxy.enabled + name: kubeAdmProxy + repository: file://./charts/kubeAdmProxy +- condition: kubeAdmScheduler.enabled + name: kubeAdmScheduler + repository: file://./charts/kubeAdmScheduler +- condition: prometheus-adapter.enabled + name: prometheus-adapter + repository: file://./charts/prometheus-adapter +- condition: nodeExporter.enabled + name: prometheus-node-exporter + repository: file://./charts/prometheus-node-exporter +- condition: rke2ControllerManager.enabled + name: rke2ControllerManager + repository: file://./charts/rke2ControllerManager +- condition: rke2Etcd.enabled + name: rke2Etcd + repository: file://./charts/rke2Etcd +- condition: rke2IngressNginx.enabled + name: rke2IngressNginx + repository: file://./charts/rke2IngressNginx +- condition: rke2Proxy.enabled + name: rke2Proxy + repository: file://./charts/rke2Proxy +- condition: rke2Scheduler.enabled + name: rke2Scheduler + repository: file://./charts/rke2Scheduler +- condition: rkeControllerManager.enabled + name: rkeControllerManager + repository: file://./charts/rkeControllerManager +- condition: rkeEtcd.enabled + name: rkeEtcd + repository: file://./charts/rkeEtcd +- condition: rkeIngressNginx.enabled + name: rkeIngressNginx + repository: file://./charts/rkeIngressNginx +- condition: rkeProxy.enabled + name: rkeProxy + repository: file://./charts/rkeProxy +- condition: rkeScheduler.enabled + name: rkeScheduler + repository: file://./charts/rkeScheduler +- condition: windowsExporter.enabled + name: windowsExporter + repository: file://./charts/windowsExporter +description: kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards, + and Prometheus rules combined with documentation and scripts to provide easy to + operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus + Operator. +home: https://github.com/prometheus-operator/kube-prometheus +icon: file://assets/logos/rancher-monitoring.png +keywords: +- operator +- prometheus +- kube-prometheus +kubeVersion: '>=1.19.0-0' +maintainers: +- email: andrew@quadcorps.co.uk + name: andrewgkew +- email: gianrubio@gmail.com + name: gianrubio +- email: github.gkarthiks@gmail.com + name: gkarthiks +- email: kube-prometheus-stack@sisti.pt + name: GMartinez-Sisti +- email: github@jkroepke.de + name: jkroepke +- email: scott@r6by.com + name: scottrigby +- email: miroslav.hadzhiev@gmail.com + name: Xtigyro +- email: quentin.bisson@gmail.com + name: QuentinBisson +name: rancher-monitoring +sources: +- https://github.com/prometheus-community/helm-charts +- https://github.com/prometheus-operator/kube-prometheus +type: application +version: 104.1.2-rc.1+up57.0.3 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/README.md new file mode 100644 index 0000000000..9baf58bb16 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/README.md @@ -0,0 +1,1080 @@ +# kube-prometheus-stack + +Installs the [kube-prometheus stack](https://github.com/prometheus-operator/kube-prometheus), a collection of Kubernetes manifests, [Grafana](http://grafana.com/) dashboards, and [Prometheus rules](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with [Prometheus](https://prometheus.io/) using the [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator). + +See the [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) README for details about components, dashboards, and alerts. + +_Note: This chart was formerly named `prometheus-operator` chart, now renamed to more clearly reflect that it installs the `kube-prometheus` project stack, within which Prometheus Operator is only one component._ + +## Prerequisites + +- Kubernetes 1.19+ +- Helm 3+ + +## Get Helm Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Helm Chart + +```console +helm install [RELEASE_NAME] prometheus-community/kube-prometheus-stack +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Dependencies + +By default this chart installs additional, dependent charts: + +- [prometheus-community/kube-state-metrics](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics) +- [prometheus-community/prometheus-node-exporter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-node-exporter) +- [grafana/grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana) + +To disable dependencies during installation, see [multiple releases](#multiple-releases) below. + +_See [helm dependency](https://helm.sh/docs/helm/helm_dependency/) for command documentation._ + +## Uninstall Helm Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +CRDs created by this chart are not removed by default and should be manually cleaned up: + +```console +kubectl delete crd alertmanagerconfigs.monitoring.coreos.com +kubectl delete crd alertmanagers.monitoring.coreos.com +kubectl delete crd podmonitors.monitoring.coreos.com +kubectl delete crd probes.monitoring.coreos.com +kubectl delete crd prometheusagents.monitoring.coreos.com +kubectl delete crd prometheuses.monitoring.coreos.com +kubectl delete crd prometheusrules.monitoring.coreos.com +kubectl delete crd scrapeconfigs.monitoring.coreos.com +kubectl delete crd servicemonitors.monitoring.coreos.com +kubectl delete crd thanosrulers.monitoring.coreos.com +``` + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack +``` + +With Helm v3, CRDs created by this chart are not updated by default and should be manually updated. +Consult also the [Helm Documentation on CRDs](https://helm.sh/docs/chart_best_practices/custom_resource_definitions). + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### Upgrading an existing Release to a new major version + +A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an incompatible breaking change needing manual actions. + +### From 56.x to 57.x + +This version upgrades Prometheus-Operator to v0.72.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 55.x to 56.x + +This version upgrades Prometheus-Operator to v0.71.0, Prometheus to 2.49.1 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 54.x to 55.x + +This version upgrades Prometheus-Operator to v0.70.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 53.x to 54.x + +Grafana Helm Chart has bumped to version 7 + +Please note Grafana Helm Chart [changelog](https://github.com/grafana/helm-charts/tree/main/charts/grafana#to-700). + +### From 52.x to 53.x + +This version upgrades Prometheus-Operator to v0.69.1, Prometheus to 2.47.2 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 51.x to 52.x + +This includes the ability to select between using existing secrets or create new secret objects for various thanos config. The defaults have not changed but if you were setting: + +- `thanosRuler.thanosRulerSpec.alertmanagersConfig` or +- `thanosRuler.thanosRulerSpec.objectStorageConfig` or +- `thanosRuler.thanosRulerSpec.queryConfig` or +- `prometheus.prometheusSpec.thanos.objectStorageConfig` + +you will have to need to set `existingSecret` or `secret` based on your requirement + +For instance, the `thanosRuler.thanosRulerSpec.alertmanagersConfig` used to be configured as follow: + +```yaml +thanosRuler: + thanosRulerSpec: + alertmanagersConfig: + alertmanagers: + - api_version: v2 + http_config: + basic_auth: + username: some_user + password: some_pass + static_configs: + - alertmanager.thanos.io + scheme: http + timeout: 10s +``` + +But it now moved to: + +```yaml +thanosRuler: + thanosRulerSpec: + alertmanagersConfig: + secret: + alertmanagers: + - api_version: v2 + http_config: + basic_auth: + username: some_user + password: some_pass + static_configs: + - alertmanager.thanos.io + scheme: http + timeout: 10s +``` + +or the `thanosRuler.thanosRulerSpec.objectStorageConfig` used to be configured as follow: + +```yaml +thanosRuler: + thanosRulerSpec: + objectStorageConfig: + name: existing-secret-not-created-by-this-chart + key: object-storage-configs.yaml +``` + +But it now moved to: + +```yaml +thanosRuler: + thanosRulerSpec: + objectStorageConfig: + existingSecret: + name: existing-secret-not-created-by-this-chart + key: object-storage-configs.yaml +``` + +### From 50.x to 51.x + +This version upgrades Prometheus-Operator to v0.68.0, Prometheus to 2.47.0 and Thanos to v0.32.2 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 49.x to 50.x + +This version requires Kubernetes 1.19+. + +We do not expect any breaking changes in this version. + +### From 48.x to 49.x + +This version upgrades Prometheus-Operator to v0.67.1, 0, Alertmanager to v0.26.0, Prometheus to 2.46.0 and Thanos to v0.32.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 47.x to 48.x + +This version moved all CRDs into a dedicated sub-chart. No new CRDs are introduced in this version. +See [#3548](https://github.com/prometheus-community/helm-charts/issues/3548) for more context. + +We do not expect any breaking changes in this version. + +### From 46.x to 47.x + +This version upgrades Prometheus-Operator to v0.66.0 with new CRDs (PrometheusAgent and ScrapeConfig). + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 45.x to 46.x + +This version upgrades Prometheus-Operator to v0.65.1 with new CRDs (PrometheusAgent and ScrapeConfig), Prometheus to v2.44.0 and Thanos to v0.31.0. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 44.x to 45.x + +This version upgrades Prometheus-Operator to v0.63.0, Prometheus to v2.42.0 and Thanos to v0.30.2. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 43.x to 44.x + +This version upgrades Prometheus-Operator to v0.62.0, Prometheus to v2.41.0 and Thanos to v0.30.1. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +If you have explicitly set `prometheusOperator.admissionWebhooks.failurePolicy`, this value is now always used even when `.prometheusOperator.admissionWebhooks.patch.enabled` is `true` (the default). + +The values for `prometheusOperator.image.tag` & `prometheusOperator.prometheusConfigReloader.image.tag` are now empty by default and the Chart.yaml `appVersion` field is used instead. + +### From 42.x to 43.x + +This version upgrades Prometheus-Operator to v0.61.1, Prometheus to v2.40.5 and Thanos to v0.29.0. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 41.x to 42.x + +This includes the overridability of container registry for all containers at the global level using `global.imageRegistry` or per container image. The defaults have not changed but if you were using a custom image, you will have to override the registry of said custom container image before you upgrade. + +For instance, the prometheus-config-reloader used to be configured as follow: + +```yaml + image: + repository: quay.io/prometheus-operator/prometheus-config-reloader + tag: v0.60.1 + sha: "" +``` + +But it now moved to: + +```yaml + image: + registry: quay.io + repository: prometheus-operator/prometheus-config-reloader + tag: v0.60.1 + sha: "" +``` + +### From 40.x to 41.x + +This version upgrades Prometheus-Operator to v0.60.1, Prometheus to v2.39.1 and Thanos to v0.28.1. +This version also upgrades the Helm charts of kube-state-metrics to 4.20.2, prometheus-node-exporter to 4.3.0 and Grafana to 6.40.4. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +This version splits kubeScheduler recording and altering rules in separate config values. +Instead of `defaultRules.rules.kubeScheduler` the 2 new variables `defaultRules.rules.kubeSchedulerAlerting` and `defaultRules.rules.kubeSchedulerRecording` are used. + +### From 39.x to 40.x + +This version upgrades Prometheus-Operator to v0.59.1, Prometheus to v2.38.0, kube-state-metrics to v2.6.0 and Thanos to v0.28.0. +This version also upgrades the Helm charts of kube-state-metrics to 4.18.0 and prometheus-node-exporter to 4.2.0. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +Starting from prometheus-node-exporter version 4.0.0, the `node exporter` chart is using the [Kubernetes recommended labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/). Therefore you have to delete the daemonset before you upgrade. + +```console +kubectl delete daemonset -l app=prometheus-node-exporter +helm upgrade -i kube-prometheus-stack prometheus-community/kube-prometheus-stack +``` + +If you use your own custom [ServiceMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor) or [PodMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#podmonitor), please ensure to upgrade their `selector` fields accordingly to the new labels. + +### From 38.x to 39.x + +This upgraded prometheus-operator to v0.58.0 and prometheus to v2.37.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 37.x to 38.x + +Reverted one of the default metrics relabelings for cAdvisor added in 36.x, due to it breaking container_network_* and various other statistics. If you do not want this change, you will need to override the `kubelet.cAdvisorMetricRelabelings`. + +### From 36.x to 37.x + +This includes some default metric relabelings for cAdvisor and apiserver metrics to reduce cardinality. If you do not want these defaults, you will need to override the `kubeApiServer.metricRelabelings` and or `kubelet.cAdvisorMetricRelabelings`. + +### From 35.x to 36.x + +This upgraded prometheus-operator to v0.57.0 and prometheus to v2.36.1 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 34.x to 35.x + +This upgraded prometheus-operator to v0.56.0 and prometheus to v2.35.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 33.x to 34.x + +This upgrades to prometheus-operator to v0.55.0 and prometheus to v2.33.5. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 32.x to 33.x + +This upgrades the prometheus-node-exporter Chart to v3.0.0. Please review the changes to this subchart if you make customizations to hostMountPropagation. + +### From 31.x to 32.x + +This upgrades to prometheus-operator to v0.54.0 and prometheus to v2.33.1. It also changes the default for `grafana.serviceMonitor.enabled` to `true. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 30.x to 31.x + +This version removes the built-in grafana ServiceMonitor and instead relies on the ServiceMonitor of the sub-chart. +`grafana.serviceMonitor.enabled` must be set instead of `grafana.serviceMonitor.selfMonitor` and the old ServiceMonitor may +need to be manually cleaned up after deploying the new release. + +### From 29.x to 30.x + +This version updates kube-state-metrics to 4.3.0 and uses the new option `kube-state-metrics.releaseLabel=true` which adds the "release" label to kube-state-metrics labels, making scraping of the metrics by kube-prometheus-stack work out of the box again, independent of the used kube-prometheus-stack release name. If you already set the "release" label via `kube-state-metrics.customLabels` you might have to remove that and use it via the new option. + +### From 28.x to 29.x + +This version makes scraping port for kube-controller-manager and kube-scheduler dynamic to reflect changes to default serving ports +for those components in Kubernetes versions v1.22 and v1.23 respectively. + +If you deploy on clusters using version v1.22+, kube-controller-manager will be scraped over HTTPS on port 10257. + +If you deploy on clusters running version v1.23+, kube-scheduler will be scraped over HTTPS on port 10259. + +### From 27.x to 28.x + +This version disables PodSecurityPolicies by default because they are deprecated in Kubernetes 1.21 and will be removed in Kubernetes 1.25. + +If you are using PodSecurityPolicies you can enable the previous behaviour by setting `kube-state-metrics.podSecurityPolicy.enabled`, `prometheus-node-exporter.rbac.pspEnabled`, `grafana.rbac.pspEnabled` and `global.rbac.pspEnabled` to `true`. + +### From 26.x to 27.x + +This version splits prometheus-node-exporter chart recording and altering rules in separate config values. +Instead of `defaultRules.rules.node` the 2 new variables `defaultRules.rules.nodeExporterAlerting` and `defaultRules.rules.nodeExporterRecording` are used. + +Also the following defaultRules.rules has been removed as they had no effect: `kubeApiserverError`, `kubePrometheusNodeAlerting`, `kubernetesAbsent`, `time`. + +The ability to set a rubookUrl via `defaultRules.rules.rubookUrl` was reintroduced. + +### From 25.x to 26.x + +This version enables the prometheus-node-exporter subchart servicemonitor by default again, by setting `prometheus-node-exporter.prometheus.monitor.enabled` to `true`. + +### From 24.x to 25.x + +This version upgrade to prometheus-operator v0.53.1. It removes support for setting a runbookUrl, since the upstream format for runbooks changed. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 23.x to 24.x + +The custom `ServiceMonitor` for the _kube-state-metrics_ & _prometheus-node-exporter_ charts have been removed in favour of the built-in sub-chart `ServiceMonitor`; for both sub-charts this means that `ServiceMonitor` customisations happen via the values passed to the chart. If you haven't directly customised this behaviour then there are no changes required to upgrade, but if you have please read the following. + +For _kube-state-metrics_ the `ServiceMonitor` customisation is now set via `kube-state-metrics.prometheus.monitor` and the `kubeStateMetrics.serviceMonitor.selfMonitor.enabled` value has moved to `kube-state-metrics.selfMonitor.enabled`. + +For _prometheus-node-exporter_ the `ServiceMonitor` customisation is now set via `prometheus-node-exporter.prometheus.monitor` and the `nodeExporter.jobLabel` values has moved to `prometheus-node-exporter.prometheus.monitor.jobLabel`. + +### From 22.x to 23.x + +Port names have been renamed for Istio's +[explicit protocol selection](https://istio.io/latest/docs/ops/configuration/traffic-management/protocol-selection/#explicit-protocol-selection). + +| | old value | new value | +|-|-----------|-----------| +| `alertmanager.alertmanagerSpec.portName` | `web` | `http-web` | +| `grafana.service.portName` | `service` | `http-web` | +| `prometheus-node-exporter.service.portName` | `metrics` (hardcoded) | `http-metrics` | +| `prometheus.prometheusSpec.portName` | `web` | `http-web` | + +### From 21.x to 22.x + +Due to the upgrade of the `kube-state-metrics` chart, removal of its deployment/stateful needs to done manually prior to upgrading: + +```console +kubectl delete deployments.apps -l app.kubernetes.io/instance=prometheus-operator,app.kubernetes.io/name=kube-state-metrics --cascade=orphan +``` + +or if you use autosharding: + +```console +kubectl delete statefulsets.apps -l app.kubernetes.io/instance=prometheus-operator,app.kubernetes.io/name=kube-state-metrics --cascade=orphan +``` + +### From 20.x to 21.x + +The config reloader values have been refactored. All the values have been moved to the key `prometheusConfigReloader` and the limits and requests can now be set separately. + +### From 19.x to 20.x + +Version 20 upgrades prometheus-operator from 0.50.x to 0.52.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 18.x to 19.x + +`kubeStateMetrics.serviceMonitor.namespaceOverride` was removed. +Please use `kube-state-metrics.namespaceOverride` instead. + +### From 17.x to 18.x + +Version 18 upgrades prometheus-operator from 0.49.x to 0.50.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 16.x to 17.x + +Version 17 upgrades prometheus-operator from 0.48.x to 0.49.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 15.x to 16.x + +Version 16 upgrades kube-state-metrics to v2.0.0. This includes changed command-line arguments and removed metrics, see this [blog post](https://kubernetes.io/blog/2021/04/13/kube-state-metrics-v-2-0/). This version also removes Grafana dashboards that supported Kubernetes 1.14 or earlier. + +### From 14.x to 15.x + +Version 15 upgrades prometheus-operator from 0.46.x to 0.47.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 13.x to 14.x + +Version 14 upgrades prometheus-operator from 0.45.x to 0.46.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 12.x to 13.x + +Version 13 upgrades prometheus-operator from 0.44.x to 0.45.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +``` + +### From 11.x to 12.x + +Version 12 upgrades prometheus-operator from 0.43.x to 0.44.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.44/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +``` + +The chart was migrated to support only helm v3 and later. + +### From 10.x to 11.x + +Version 11 upgrades prometheus-operator from 0.42.x to 0.43.x. Starting with 0.43.x an additional `AlertmanagerConfigs` CRD is introduced. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.43/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +``` + +Version 11 removes the deprecated tlsProxy via ghostunnel in favor of native TLS support the prometheus-operator gained with v0.39.0. + +### From 9.x to 10.x + +Version 10 upgrades prometheus-operator from 0.38.x to 0.42.x. Starting with 0.40.x an additional `Probes` CRD is introduced. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.42/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +``` + +### From 8.x to 9.x + +Version 9 of the helm chart removes the existing `additionalScrapeConfigsExternal` in favour of `additionalScrapeConfigsSecret`. This change lets users specify the secret name and secret key to use for the additional scrape configuration of prometheus. This is useful for users that have prometheus-operator as a subchart and also have a template that creates the additional scrape configuration. + +### From 7.x to 8.x + +Due to new template functions being used in the rules in version 8.x.x of the chart, an upgrade to Prometheus Operator and Prometheus is necessary in order to support them. First, upgrade to the latest version of 7.x.x + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack --version 7.5.0 +``` + +Then upgrade to 8.x.x + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack --version [8.x.x] +``` + +Minimal recommended Prometheus version for this chart release is `2.12.x` + +### From 6.x to 7.x + +Due to a change in grafana subchart, version 7.x.x now requires Helm >= 2.12.0. + +### From 5.x to 6.x + +Due to a change in deployment labels of kube-state-metrics, the upgrade requires `helm upgrade --force` in order to re-create the deployment. If this is not done an error will occur indicating that the deployment cannot be modified: + +```console +invalid: spec.selector: Invalid value: v1.LabelSelector{MatchLabels:map[string]string{"app.kubernetes.io/name":"kube-state-metrics"}, MatchExpressions:[]v1.LabelSelectorRequirement(nil)}: field is immutable +``` + +If this error has already been encountered, a `helm history` command can be used to determine which release has worked, then `helm rollback` to the release, then `helm upgrade --force` to this new one + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments: + +```console +helm show values prometheus-community/kube-prometheus-stack +``` + +You may also run `helm show values` on this chart's [dependencies](#dependencies) for additional options. + +### Rancher Monitoring Configuration + +The following table shows values exposed by Rancher Monitoring's additions to the chart: + +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `nameOverride` | Provide a name that should be used instead of the chart name when naming all resources deployed by this chart |`"rancher-monitoring"`| +| `namespaceOverride` | Override the deployment namespace | `"cattle-monitoring-system"` | +| `global.rbac.userRoles.create` | Create default user ClusterRoles to allow users to interact with Prometheus CRs, ConfigMaps, and Secrets | `true` | +| `global.rbac.userRoles.aggregateToDefaultRoles` | Aggregate default user ClusterRoles into default k8s ClusterRoles | `true` | +| `prometheus-adapter.enabled` | Whether to install [prometheus-adapter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter) within the cluster | `true` | +| `prometheus-adapter.prometheus.url` | A URL pointing to the Prometheus deployment within your cluster. The default value is set based on the assumption that you plan to deploy the default Prometheus instance from this chart where `.Values.namespaceOverride=cattle-monitoring-system` and `.Values.nameOverride=rancher-monitoring` | `http://rancher-monitoring-prometheus.cattle-monitoring-system.svc` | +| `prometheus-adapter.prometheus.port` | The port on the Prometheus deployment that Prometheus Adapter can make requests to | `9090` | +| `prometheus.prometheusSpec.ignoreNamespaceSelectors` | Ignore NamespaceSelector settings from the PodMonitor and ServiceMonitor configs. If true, PodMonitors and ServiceMonitors can only discover Pods and Services within the namespace they are deployed into | `false` | + +The following values are enabled for different distributions via [rancher-pushprox](https://github.com/rancher/dev-charts/tree/master/packages/rancher-pushprox). See the rancher-pushprox `README.md` for more information on what all values can be configured for the PushProxy chart. + +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `rkeControllerManager.enabled` | Create a PushProx installation for monitoring kube-controller-manager metrics in RKE clusters | `false` | +| `rkeScheduler.enabled` | Create a PushProx installation for monitoring kube-scheduler metrics in RKE clusters | `false` | +| `rkeProxy.enabled` | Create a PushProx installation for monitoring kube-proxy metrics in RKE clusters | `false` | +| `rkeIngressNginx.enabled` | Create a PushProx installation for monitoring ingress-nginx metrics in RKE clusters | `false` | +| `rkeEtcd.enabled` | Create a PushProx installation for monitoring etcd metrics in RKE clusters | `false` | +| `rke2IngressNginx.enabled` | Create a PushProx installation for monitoring ingress-nginx metrics in RKE2 clusters | `false` | +| `k3sServer.enabled` | Create a PushProx installation for monitoring k3s-server metrics (accounts for kube-controller-manager, kube-scheduler, and kube-proxy metrics) in k3s clusters | `false` | +| `kubeAdmControllerManager.enabled` | Create a PushProx installation for monitoring kube-controller-manager metrics in kubeAdm clusters | `false` | +| `kubeAdmScheduler.enabled` | Create a PushProx installation for monitoring kube-scheduler metrics in kubeAdm clusters | `false` | +| `kubeAdmProxy.enabled` | Create a PushProx installation for monitoring kube-proxy metrics in kubeAdm clusters | `false` | +| `kubeAdmEtcd.enabled` | Create a PushProx installation for monitoring etcd metrics in kubeAdm clusters | `false` | + + +### Multiple releases + +The same chart can be used to run multiple Prometheus instances in the same cluster if required. To achieve this, it is necessary to run only one instance of prometheus-operator and a pair of alertmanager pods for an HA configuration, while all other components need to be disabled. To disable a dependency during installation, set `kubeStateMetrics.enabled`, `nodeExporter.enabled` and `grafana.enabled` to `false`. + +## Work-Arounds for Known Issues + +### Running on private GKE clusters + +When Google configure the control plane for private clusters, they automatically configure VPC peering between your Kubernetes cluster’s network and a separate Google managed project. In order to restrict what Google are able to access within your cluster, the firewall rules configured restrict access to your Kubernetes pods. This means that in order to use the webhook component with a GKE private cluster, you must configure an additional firewall rule to allow the GKE control plane access to your webhook pod. + +You can read more information on how to add firewall rules for the GKE control plane nodes in the [GKE docs](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#add_firewall_rules) + +Alternatively, you can disable the hooks by setting `prometheusOperator.admissionWebhooks.enabled=false`. + +## PrometheusRules Admission Webhooks + +With Prometheus Operator version 0.30+, the core Prometheus Operator pod exposes an endpoint that will integrate with the `validatingwebhookconfiguration` Kubernetes feature to prevent malformed rules from being added to the cluster. + +### How the Chart Configures the Hooks + +A validating and mutating webhook configuration requires the endpoint to which the request is sent to use TLS. It is possible to set up custom certificates to do this, but in most cases, a self-signed certificate is enough. The setup of this component requires some more complex orchestration when using helm. The steps are created to be idempotent and to allow turning the feature on and off without running into helm quirks. + +1. A pre-install hook provisions a certificate into the same namespace using a format compatible with provisioning using end user certificates. If the certificate already exists, the hook exits. +2. The prometheus operator pod is configured to use a TLS proxy container, which will load that certificate. +3. Validating and Mutating webhook configurations are created in the cluster, with their failure mode set to Ignore. This allows rules to be created by the same chart at the same time, even though the webhook has not yet been fully set up - it does not have the correct CA field set. +4. A post-install hook reads the CA from the secret created by step 1 and patches the Validating and Mutating webhook configurations. This process will allow a custom CA provisioned by some other process to also be patched into the webhook configurations. The chosen failure policy is also patched into the webhook configurations + +### Alternatives + +It should be possible to use [jetstack/cert-manager](https://github.com/jetstack/cert-manager) if a more complete solution is required, but it has not been tested. + +You can enable automatic self-signed TLS certificate provisioning via cert-manager by setting the `prometheusOperator.admissionWebhooks.certManager.enabled` value to true. + +### Limitations + +Because the operator can only run as a single pod, there is potential for this component failure to cause rule deployment failure. Because this risk is outweighed by the benefit of having validation, the feature is enabled by default. + +## Developing Prometheus Rules and Grafana Dashboards + +This chart Grafana Dashboards and Prometheus Rules are just a copy from [prometheus-operator/prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) and other sources, synced (with alterations) by scripts in [hack](hack) folder. In order to introduce any changes you need to first [add them to the original repository](https://github.com/prometheus-operator/kube-prometheus/blob/main/docs/customizations/developing-prometheus-rules-and-grafana-dashboards.md) and then sync there by scripts. + +## Further Information + +For more in-depth documentation of configuration options meanings, please see + +- [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) +- [Prometheus](https://prometheus.io/docs/introduction/overview/) +- [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana#grafana-helm-chart) + +## prometheus.io/scrape + +The prometheus operator does not support annotation-based discovery of services, using the `PodMonitor` or `ServiceMonitor` CRD in its place as they provide far more configuration options. +For information on how to use PodMonitors/ServiceMonitors, please see the documentation on the `prometheus-operator/prometheus-operator` documentation here: + +- [ServiceMonitors](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/getting-started.md#include-servicemonitors) +- [PodMonitors](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/getting-started.md#include-podmonitors) +- [Running Exporters](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/running-exporters.md) + +By default, Prometheus discovers PodMonitors and ServiceMonitors within its namespace, that are labeled with the same release tag as the prometheus-operator release. +Sometimes, you may need to discover custom PodMonitors/ServiceMonitors, for example used to scrape data from third-party applications. +An easy way of doing this, without compromising the default PodMonitors/ServiceMonitors discovery, is allowing Prometheus to discover all PodMonitors/ServiceMonitors within its namespace, without applying label filtering. +To do so, you can set `prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues` and `prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues` to `false`. + +## Migrating from stable/prometheus-operator chart + +## Zero downtime + +Since `kube-prometheus-stack` is fully compatible with the `stable/prometheus-operator` chart, a migration without downtime can be achieved. +However, the old name prefix needs to be kept. If you want the new name please follow the step by step guide below (with downtime). + +You can override the name to achieve this: + +```console +helm upgrade prometheus-operator prometheus-community/kube-prometheus-stack -n monitoring --reuse-values --set nameOverride=prometheus-operator +``` + +**Note**: It is recommended to run this first with `--dry-run --debug`. + +## Redeploy with new name (downtime) + +If the **prometheus-operator** values are compatible with the new **kube-prometheus-stack** chart, please follow the below steps for migration: + +> The guide presumes that chart is deployed in `monitoring` namespace and the deployments are running there. If in other namespace, please replace the `monitoring` to the deployed namespace. + +1. Patch the PersistenceVolume created/used by the prometheus-operator chart to `Retain` claim policy: + + ```console + kubectl patch pv/ -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' + ``` + + **Note:** To execute the above command, the user must have a cluster wide permission. Please refer [Kubernetes RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) + +2. Uninstall the **prometheus-operator** release and delete the existing PersistentVolumeClaim, and verify PV become Released. + + ```console + helm uninstall prometheus-operator -n monitoring + kubectl delete pvc/ -n monitoring + ``` + + Additionally, you have to manually remove the remaining `prometheus-operator-kubelet` service. + + ```console + kubectl delete service/prometheus-operator-kubelet -n kube-system + ``` + + You can choose to remove all your existing CRDs (ServiceMonitors, Podmonitors, etc.) if you want to. + +3. Remove current `spec.claimRef` values to change the PV's status from Released to Available. + + ```console + kubectl patch pv/ --type json -p='[{"op": "remove", "path": "/spec/claimRef"}]' -n monitoring + ``` + +**Note:** To execute the above command, the user must have a cluster wide permission. Please refer to [Kubernetes RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) + +After these steps, proceed to a fresh **kube-prometheus-stack** installation and make sure the current release of **kube-prometheus-stack** matching the `volumeClaimTemplate` values in the `values.yaml`. + +The binding is done via matching a specific amount of storage requested and with certain access modes. + +For example, if you had storage specified as this with **prometheus-operator**: + +```yaml +volumeClaimTemplate: + spec: + storageClassName: gp2 + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 50Gi +``` + +You have to specify matching `volumeClaimTemplate` with 50Gi storage and `ReadWriteOnce` access mode. + +Additionally, you should check the current AZ of your legacy installation's PV, and configure the fresh release to use the same AZ as the old one. If the pods are in a different AZ than the PV, the release will fail to bind the existing one, hence creating a new PV. + +This can be achieved either by specifying the labels through `values.yaml`, e.g. setting `prometheus.prometheusSpec.nodeSelector` to: + +```yaml +nodeSelector: + failure-domain.beta.kubernetes.io/zone: east-west-1a +``` + +or passing these values as `--set` overrides during installation. + +The new release should now re-attach your previously released PV with its content. + +## Migrating from coreos/prometheus-operator chart + +The multiple charts have been combined into a single chart that installs prometheus operator, prometheus, alertmanager, grafana as well as the multitude of exporters necessary to monitor a cluster. + +There is no simple and direct migration path between the charts as the changes are extensive and intended to make the chart easier to support. + +The capabilities of the old chart are all available in the new chart, including the ability to run multiple prometheus instances on a single cluster - you will need to disable the parts of the chart you do not wish to deploy. + +You can check out the tickets for this change [here](https://github.com/prometheus-operator/prometheus-operator/issues/592) and [here](https://github.com/helm/charts/pull/6765). + +### High-level overview of Changes + +#### Added dependencies + +The chart has added 3 [dependencies](#dependencies). + +- Node-Exporter, Kube-State-Metrics: These components are loaded as dependencies into the chart, and are relatively simple components +- Grafana: The Grafana chart is more feature-rich than this chart - it contains a sidecar that is able to load data sources and dashboards from configmaps deployed into the same cluster. For more information check out the [documentation for the chart](https://github.com/grafana/helm-charts/blob/main/charts/grafana/README.md) + +#### Kubelet Service + +Because the kubelet service has a new name in the chart, make sure to clean up the old kubelet service in the `kube-system` namespace to prevent counting container metrics twice. + +#### Persistent Volumes + +If you would like to keep the data of the current persistent volumes, it should be possible to attach existing volumes to new PVCs and PVs that are created using the conventions in the new chart. For example, in order to use an existing Azure disk for a helm release called `prometheus-migration` the following resources can be created: + +```yaml +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pvc-prometheus-migration-prometheus-0 +spec: + accessModes: + - ReadWriteOnce + azureDisk: + cachingMode: None + diskName: pvc-prometheus-migration-prometheus-0 + diskURI: /subscriptions/f5125d82-2622-4c50-8d25-3f7ba3e9ac4b/resourceGroups/sample-migration-resource-group/providers/Microsoft.Compute/disks/pvc-prometheus-migration-prometheus-0 + fsType: "" + kind: Managed + readOnly: false + capacity: + storage: 1Gi + persistentVolumeReclaimPolicy: Delete + storageClassName: prometheus + volumeMode: Filesystem +``` + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + app.kubernetes.io/name: prometheus + prometheus: prometheus-migration-prometheus + name: prometheus-prometheus-migration-prometheus-db-prometheus-prometheus-migration-prometheus-0 + namespace: monitoring +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: prometheus + volumeMode: Filesystem + volumeName: pvc-prometheus-migration-prometheus-0 +``` + +The PVC will take ownership of the PV and when you create a release using a persistent volume claim template it will use the existing PVCs as they match the naming convention used by the chart. For other cloud providers similar approaches can be used. + +#### KubeProxy + +The metrics bind address of kube-proxy is default to `127.0.0.1:10249` that prometheus instances **cannot** access to. You should expose metrics by changing `metricsBindAddress` field value to `0.0.0.0:10249` if you want to collect them. + +Depending on the cluster, the relevant part `config.conf` will be in ConfigMap `kube-system/kube-proxy` or `kube-system/kube-proxy-config`. For example: + +```console +kubectl -n kube-system edit cm kube-proxy +``` + +```yaml +apiVersion: v1 +data: + config.conf: |- + apiVersion: kubeproxy.config.k8s.io/v1alpha1 + kind: KubeProxyConfiguration + # ... + # metricsBindAddress: 127.0.0.1:10249 + metricsBindAddress: 0.0.0.0:10249 + # ... + kubeconfig.conf: |- + # ... +kind: ConfigMap +metadata: + labels: + app: kube-proxy + name: kube-proxy + namespace: kube-system +``` diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/app-README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/app-README.md new file mode 100644 index 0000000000..3920854384 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/app-README.md @@ -0,0 +1,46 @@ +# Rancher Monitoring and Alerting + + This chart is based on the upstream [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) chart. The chart deploys [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) and its CRDs along with [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana), [Prometheus Adapter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter) and additional charts / Kubernetes manifests to gather metrics. It allows users to monitor their Kubernetes clusters, view metrics in Grafana dashboards, and set up alerts and notifications. + +For more information on how to use the feature, refer to our [docs](https://rancher.com/docs/rancher/v2.x/en/monitoring-alerting/v2.5/). + +The chart installs the following components: + +- [Prometheus Operator](https://github.com/coreos/prometheus-operator) - The operator provides easy monitoring definitions for Kubernetes services, manages [Prometheus](https://prometheus.io/) and [AlertManager](https://prometheus.io/docs/alerting/latest/alertmanager/) instances, and adds default scrape targets for some Kubernetes components. +- [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus/) - A collection of community-curated Kubernetes manifests, Grafana Dashboards, and PrometheusRules that deploy a default end-to-end cluster monitoring configuration. +- [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana) - Grafana allows a user to create / view dashboards based on the cluster metrics collected by Prometheus. +- [node-exporter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-node-exporter) / [kube-state-metrics](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics) / [rancher-pushprox](https://github.com/rancher/charts/tree/dev-v2.7/packages/rancher-monitoring/rancher-pushprox/charts) - These charts monitor various Kubernetes components across different Kubernetes cluster types. +- [Prometheus Adapter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter) - The adapter allows a user to expose custom metrics, resource metrics, and external metrics on the default [Prometheus](https://prometheus.io/) instance to the Kubernetes API Server. + +For more information, review the Helm README of this chart. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. +​ +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Upgrading from 100.0.0+up16.6.0 to 100.1.0+up19.0.3 + +### Noticeable changes: +Grafana: +- `sidecar.dashboards.searchNamespace`, `sidecar.datasources.searchNamespace` and `sidecar.notifiers.searchNamespace` support a list of namespaces now. + +Kube-state-metrics +- the type of `collectors` is changed from Dictionary to List. +- `kubeStateMetrics.serviceMonitor.namespaceOverride` was replaced by `kube-state-metrics.namespaceOverride`. + +### Known issues: +- Occasionally, the upgrade fails with errors related to the webhook `prometheusrulemutate.monitoring.coreos.com`. This is a known issue in the upstream, and the workaround is to trigger the upgrade one more time. [32416](https://github.com/rancher/rancher/issues/32416#issuecomment-828881726) diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/.helmignore new file mode 100644 index 0000000000..8cade1318f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.vscode +.project +.idea/ +*.tmproj +OWNERS diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/Chart.yaml new file mode 100644 index 0000000000..ff6bcb26aa --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/Chart.yaml @@ -0,0 +1,39 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/grafana/helm-charts + - name: Upstream Project + url: https://github.com/grafana/grafana + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-grafana +apiVersion: v2 +appVersion: 10.4.1 +description: The leading tool for querying and visualizing time series and metrics. +home: https://grafana.com +icon: https://artifacthub.io/image/b4fed1a7-6c8f-4945-b99d-096efa3e4116 +keywords: +- monitoring +- metric +kubeVersion: '>=1.26.0-0' +maintainers: +- email: zanhsieh@gmail.com + name: zanhsieh +- email: rluckie@cisco.com + name: rtluckie +- email: maor.friedman@redhat.com + name: maorfr +- email: miroslav.hadzhiev@gmail.com + name: Xtigyro +- email: mail@torstenwalter.de + name: torstenwalter +name: grafana +sources: +- https://github.com/grafana/grafana +- https://github.com/grafana/helm-charts +type: application +version: 7.3.11 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/README.md new file mode 100644 index 0000000000..0ff07f297d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/README.md @@ -0,0 +1,770 @@ +# Grafana Helm Chart + +* Installs the web dashboarding system [Grafana](http://grafana.org/) + +## Get Repo Info + +```console +helm repo add grafana https://grafana.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```console +helm install my-release grafana/grafana +``` + +## Uninstalling the Chart + +To uninstall/delete the my-release deployment: + +```console +helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Upgrading an existing Release to a new major version + +A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an +incompatible breaking change needing manual actions. + +### To 4.0.0 (And 3.12.1) + +This version requires Helm >= 2.12.0. + +### To 5.0.0 + +You have to add --force to your helm upgrade command as the labels of the chart have changed. + +### To 6.0.0 + +This version requires Helm >= 3.1.0. + +### To 7.0.0 + +For consistency with other Helm charts, the `global.image.registry` parameter was renamed +to `global.imageRegistry`. If you were not previously setting `global.image.registry`, no action +is required on upgrade. If you were previously setting `global.image.registry`, you will +need to instead set `global.imageRegistry`. + +## Configuration + +| Parameter | Description | Default | +|-------------------------------------------|-----------------------------------------------|---------------------------------------------------------| +| `replicas` | Number of nodes | `1` | +| `podDisruptionBudget.minAvailable` | Pod disruption minimum available | `nil` | +| `podDisruptionBudget.maxUnavailable` | Pod disruption maximum unavailable | `nil` | +| `podDisruptionBudget.apiVersion` | Pod disruption apiVersion | `nil` | +| `deploymentStrategy` | Deployment strategy | `{ "type": "RollingUpdate" }` | +| `livenessProbe` | Liveness Probe settings | `{ "httpGet": { "path": "/api/health", "port": 3000 } "initialDelaySeconds": 60, "timeoutSeconds": 30, "failureThreshold": 10 }` | +| `readinessProbe` | Readiness Probe settings | `{ "httpGet": { "path": "/api/health", "port": 3000 } }`| +| `securityContext` | Deployment securityContext | `{"runAsUser": 472, "runAsGroup": 472, "fsGroup": 472}` | +| `priorityClassName` | Name of Priority Class to assign pods | `nil` | +| `image.registry` | Image registry | `docker.io` | +| `image.repository` | Image repository | `grafana/grafana` | +| `image.tag` | Overrides the Grafana image tag whose default is the chart appVersion (`Must be >= 5.0.0`) | `` | +| `image.sha` | Image sha (optional) | `` | +| `image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `image.pullSecrets` | Image pull secrets (can be templated) | `[]` | +| `service.enabled` | Enable grafana service | `true` | +| `service.type` | Kubernetes service type | `ClusterIP` | +| `service.port` | Kubernetes port where service is exposed | `80` | +| `service.portName` | Name of the port on the service | `service` | +| `service.appProtocol` | Adds the appProtocol field to the service | `` | +| `service.targetPort` | Internal service is port | `3000` | +| `service.nodePort` | Kubernetes service nodePort | `nil` | +| `service.annotations` | Service annotations (can be templated) | `{}` | +| `service.labels` | Custom labels | `{}` | +| `service.clusterIP` | internal cluster service IP | `nil` | +| `service.loadBalancerIP` | IP address to assign to load balancer (if supported) | `nil` | +| `service.loadBalancerSourceRanges` | list of IP CIDRs allowed access to lb (if supported) | `[]` | +| `service.externalIPs` | service external IP addresses | `[]` | +| `service.externalTrafficPolicy` | change the default externalTrafficPolicy | `nil` | +| `headlessService` | Create a headless service | `false` | +| `extraExposePorts` | Additional service ports for sidecar containers| `[]` | +| `hostAliases` | adds rules to the pod's /etc/hosts | `[]` | +| `ingress.enabled` | Enables Ingress | `false` | +| `ingress.annotations` | Ingress annotations (values are templated) | `{}` | +| `ingress.labels` | Custom labels | `{}` | +| `ingress.path` | Ingress accepted path | `/` | +| `ingress.pathType` | Ingress type of path | `Prefix` | +| `ingress.hosts` | Ingress accepted hostnames | `["chart-example.local"]` | +| `ingress.extraPaths` | Ingress extra paths to prepend to every host configuration. Useful when configuring [custom actions with AWS ALB Ingress Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.6/guide/ingress/annotations/#actions). Requires `ingress.hosts` to have one or more host entries. | `[]` | +| `ingress.tls` | Ingress TLS configuration | `[]` | +| `ingress.ingressClassName` | Ingress Class Name. MAY be required for Kubernetes versions >= 1.18 | `""` | +| `resources` | CPU/Memory resource requests/limits | `{}` | +| `nodeSelector` | Node labels for pod assignment | `{}` | +| `tolerations` | Toleration labels for pod assignment | `[]` | +| `affinity` | Affinity settings for pod assignment | `{}` | +| `extraInitContainers` | Init containers to add to the grafana pod | `{}` | +| `extraContainers` | Sidecar containers to add to the grafana pod | `""` | +| `extraContainerVolumes` | Volumes that can be mounted in sidecar containers | `[]` | +| `extraLabels` | Custom labels for all manifests | `{}` | +| `schedulerName` | Name of the k8s scheduler (other than default) | `nil` | +| `persistence.enabled` | Use persistent volume to store data | `false` | +| `persistence.type` | Type of persistence (`pvc` or `statefulset`) | `pvc` | +| `persistence.size` | Size of persistent volume claim | `10Gi` | +| `persistence.existingClaim` | Use an existing PVC to persist data (can be templated) | `nil` | +| `persistence.storageClassName` | Type of persistent volume claim | `nil` | +| `persistence.accessModes` | Persistence access modes | `[ReadWriteOnce]` | +| `persistence.annotations` | PersistentVolumeClaim annotations | `{}` | +| `persistence.finalizers` | PersistentVolumeClaim finalizers | `[ "kubernetes.io/pvc-protection" ]` | +| `persistence.extraPvcLabels` | Extra labels to apply to a PVC. | `{}` | +| `persistence.subPath` | Mount a sub dir of the persistent volume (can be templated) | `nil` | +| `persistence.inMemory.enabled` | If persistence is not enabled, whether to mount the local storage in-memory to improve performance | `false` | +| `persistence.inMemory.sizeLimit` | SizeLimit for the in-memory local storage | `nil` | +| `initChownData.enabled` | If false, don't reset data ownership at startup | true | +| `initChownData.image.registry` | init-chown-data container image registry | `docker.io` | +| `initChownData.image.repository` | init-chown-data container image repository | `busybox` | +| `initChownData.image.tag` | init-chown-data container image tag | `1.31.1` | +| `initChownData.image.sha` | init-chown-data container image sha (optional)| `""` | +| `initChownData.image.pullPolicy` | init-chown-data container image pull policy | `IfNotPresent` | +| `initChownData.resources` | init-chown-data pod resource requests & limits | `{}` | +| `schedulerName` | Alternate scheduler name | `nil` | +| `env` | Extra environment variables passed to pods | `{}` | +| `envValueFrom` | Environment variables from alternate sources. See the API docs on [EnvVarSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core) for format details. Can be templated | `{}` | +| `envFromSecret` | Name of a Kubernetes secret (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `""` | +| `envFromSecrets` | List of Kubernetes secrets (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `[]` | +| `envFromConfigMaps` | List of Kubernetes ConfigMaps (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `[]` | +| `envRenderSecret` | Sensible environment variables passed to pods and stored as secret. (passed through [tpl](https://helm.sh/docs/howto/charts_tips_and_tricks/#using-the-tpl-function)) | `{}` | +| `enableServiceLinks` | Inject Kubernetes services as environment variables. | `true` | +| `extraSecretMounts` | Additional grafana server secret mounts | `[]` | +| `extraVolumeMounts` | Additional grafana server volume mounts | `[]` | +| `extraVolumes` | Additional Grafana server volumes | `[]` | +| `automountServiceAccountToken` | Mounted the service account token on the grafana pod. Mandatory, if sidecars are enabled | `true` | +| `createConfigmap` | Enable creating the grafana configmap | `true` | +| `extraConfigmapMounts` | Additional grafana server configMap volume mounts (values are templated) | `[]` | +| `extraEmptyDirMounts` | Additional grafana server emptyDir volume mounts | `[]` | +| `plugins` | Plugins to be loaded along with Grafana | `[]` | +| `datasources` | Configure grafana datasources (passed through tpl) | `{}` | +| `alerting` | Configure grafana alerting (passed through tpl) | `{}` | +| `notifiers` | Configure grafana notifiers | `{}` | +| `dashboardProviders` | Configure grafana dashboard providers | `{}` | +| `dashboards` | Dashboards to import | `{}` | +| `dashboardsConfigMaps` | ConfigMaps reference that contains dashboards | `{}` | +| `grafana.ini` | Grafana's primary configuration | `{}` | +| `global.imageRegistry` | Global image pull registry for all images. | `null` | +| `global.imagePullSecrets` | Global image pull secrets (can be templated). Allows either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). | `[]` | +| `ldap.enabled` | Enable LDAP authentication | `false` | +| `ldap.existingSecret` | The name of an existing secret containing the `ldap.toml` file, this must have the key `ldap-toml`. | `""` | +| `ldap.config` | Grafana's LDAP configuration | `""` | +| `annotations` | Deployment annotations | `{}` | +| `labels` | Deployment labels | `{}` | +| `podAnnotations` | Pod annotations | `{}` | +| `podLabels` | Pod labels | `{}` | +| `podPortName` | Name of the grafana port on the pod | `grafana` | +| `lifecycleHooks` | Lifecycle hooks for podStart and preStop [Example](https://kubernetes.io/docs/tasks/configure-pod-container/attach-handler-lifecycle-event/#define-poststart-and-prestop-handlers) | `{}` | +| `sidecar.image.registry` | Sidecar image registry | `quay.io` | +| `sidecar.image.repository` | Sidecar image repository | `kiwigrid/k8s-sidecar` | +| `sidecar.image.tag` | Sidecar image tag | `1.26.0` | +| `sidecar.image.sha` | Sidecar image sha (optional) | `""` | +| `sidecar.imagePullPolicy` | Sidecar image pull policy | `IfNotPresent` | +| `sidecar.resources` | Sidecar resources | `{}` | +| `sidecar.securityContext` | Sidecar securityContext | `{}` | +| `sidecar.enableUniqueFilenames` | Sets the kiwigrid/k8s-sidecar UNIQUE_FILENAMES environment variable. If set to `true` the sidecar will create unique filenames where duplicate data keys exist between ConfigMaps and/or Secrets within the same or multiple Namespaces. | `false` | +| `sidecar.alerts.enabled` | Enables the cluster wide search for alerts and adds/updates/deletes them in grafana |`false` | +| `sidecar.alerts.label` | Label that config maps with alerts should have to be added | `grafana_alert` | +| `sidecar.alerts.labelValue` | Label value that config maps with alerts should have to be added | `""` | +| `sidecar.alerts.searchNamespace` | Namespaces list. If specified, the sidecar will search for alerts config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.alerts.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.alerts.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.alerts.reloadURL` | Full url of datasource configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/alerting/reload"` | +| `sidecar.alerts.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.alerts.initAlerts` | Set to true to deploy the alerts sidecar as an initContainer. This is needed if skipReload is true, to load any alerts defined at startup time. | `false` | +| `sidecar.alerts.extraMounts` | Additional alerts sidecar volume mounts. | `[]` | +| `sidecar.dashboards.enabled` | Enables the cluster wide search for dashboards and adds/updates/deletes them in grafana | `false` | +| `sidecar.dashboards.SCProvider` | Enables creation of sidecar provider | `true` | +| `sidecar.dashboards.provider.name` | Unique name of the grafana provider | `sidecarProvider` | +| `sidecar.dashboards.provider.orgid` | Id of the organisation, to which the dashboards should be added | `1` | +| `sidecar.dashboards.provider.folder` | Logical folder in which grafana groups dashboards | `""` | +| `sidecar.dashboards.provider.disableDelete` | Activate to avoid the deletion of imported dashboards | `false` | +| `sidecar.dashboards.provider.allowUiUpdates` | Allow updating provisioned dashboards from the UI | `false` | +| `sidecar.dashboards.provider.type` | Provider type | `file` | +| `sidecar.dashboards.provider.foldersFromFilesStructure` | Allow Grafana to replicate dashboard structure from filesystem. | `false` | +| `sidecar.dashboards.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.skipTlsVerify` | Set to true to skip tls verification for kube api calls | `nil` | +| `sidecar.dashboards.label` | Label that config maps with dashboards should have to be added | `grafana_dashboard` | +| `sidecar.dashboards.labelValue` | Label value that config maps with dashboards should have to be added | `""` | +| `sidecar.dashboards.folder` | Folder in the pod that should hold the collected dashboards (unless `sidecar.dashboards.defaultFolderName` is set). This path will be mounted. | `/tmp/dashboards` | +| `sidecar.dashboards.folderAnnotation` | The annotation the sidecar will look for in configmaps to override the destination folder for files | `nil` | +| `sidecar.dashboards.defaultFolderName` | The default folder name, it will create a subfolder under the `sidecar.dashboards.folder` and put dashboards in there instead | `nil` | +| `sidecar.dashboards.searchNamespace` | Namespaces list. If specified, the sidecar will search for dashboards config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.dashboards.script` | Absolute path to shell script to execute after a configmap got reloaded. | `nil` | +| `sidecar.dashboards.reloadURL` | Full url of dashboards configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/dashboards/reload"` | +| `sidecar.dashboards.skipReload` | Enabling this omits defining the REQ_USERNAME, REQ_PASSWORD, REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.dashboards.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.dashboards.extraMounts` | Additional dashboard sidecar volume mounts. | `[]` | +| `sidecar.datasources.enabled` | Enables the cluster wide search for datasources and adds/updates/deletes them in grafana |`false` | +| `sidecar.datasources.label` | Label that config maps with datasources should have to be added | `grafana_datasource` | +| `sidecar.datasources.labelValue` | Label value that config maps with datasources should have to be added | `""` | +| `sidecar.datasources.searchNamespace` | Namespaces list. If specified, the sidecar will search for datasources config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.datasources.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.datasources.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.datasources.reloadURL` | Full url of datasource configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/datasources/reload"` | +| `sidecar.datasources.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.datasources.initDatasources` | Set to true to deploy the datasource sidecar as an initContainer in addition to a container. This is needed if skipReload is true, to load any datasources defined at startup time. | `false` | +| `sidecar.notifiers.enabled` | Enables the cluster wide search for notifiers and adds/updates/deletes them in grafana | `false` | +| `sidecar.notifiers.label` | Label that config maps with notifiers should have to be added | `grafana_notifier` | +| `sidecar.notifiers.labelValue` | Label value that config maps with notifiers should have to be added | `""` | +| `sidecar.notifiers.searchNamespace` | Namespaces list. If specified, the sidecar will search for notifiers config-maps (or secrets) inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.notifiers.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.notifiers.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.notifiers.reloadURL` | Full url of notifier configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/notifications/reload"` | +| `sidecar.notifiers.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.notifiers.initNotifiers` | Set to true to deploy the notifier sidecar as an initContainer in addition to a container. This is needed if skipReload is true, to load any notifiers defined at startup time. | `false` | +| `smtp.existingSecret` | The name of an existing secret containing the SMTP credentials. | `""` | +| `smtp.userKey` | The key in the existing SMTP secret containing the username. | `"user"` | +| `smtp.passwordKey` | The key in the existing SMTP secret containing the password. | `"password"` | +| `admin.existingSecret` | The name of an existing secret containing the admin credentials (can be templated). | `""` | +| `admin.userKey` | The key in the existing admin secret containing the username. | `"admin-user"` | +| `admin.passwordKey` | The key in the existing admin secret containing the password. | `"admin-password"` | +| `serviceAccount.automountServiceAccountToken` | Automount the service account token on all pods where is service account is used | `false` | +| `serviceAccount.annotations` | ServiceAccount annotations | | +| `serviceAccount.create` | Create service account | `true` | +| `serviceAccount.labels` | ServiceAccount labels | `{}` | +| `serviceAccount.name` | Service account name to use, when empty will be set to created account if `serviceAccount.create` is set else to `default` | `` | +| `serviceAccount.nameTest` | Service account name to use for test, when empty will be set to created account if `serviceAccount.create` is set else to `default` | `nil` | +| `rbac.create` | Create and use RBAC resources | `true` | +| `rbac.namespaced` | Creates Role and Rolebinding instead of the default ClusterRole and ClusteRoleBindings for the grafana instance | `false` | +| `rbac.useExistingRole` | Set to a rolename to use existing role - skipping role creating - but still doing serviceaccount and rolebinding to the rolename set here. | `nil` | +| `rbac.pspEnabled` | Create PodSecurityPolicy (with `rbac.create`, grant roles permissions as well) | `false` | +| `rbac.pspUseAppArmor` | Enforce AppArmor in created PodSecurityPolicy (requires `rbac.pspEnabled`) | `false` | +| `rbac.extraRoleRules` | Additional rules to add to the Role | [] | +| `rbac.extraClusterRoleRules` | Additional rules to add to the ClusterRole | [] | +| `command` | Define command to be executed by grafana container at startup | `nil` | +| `args` | Define additional args if command is used | `nil` | +| `testFramework.enabled` | Whether to create test-related resources | `true` | +| `testFramework.image.registry` | `test-framework` image registry. | `docker.io` | +| `testFramework.image.repository` | `test-framework` image repository. | `bats/bats` | +| `testFramework.image.tag` | `test-framework` image tag. | `v1.4.1` | +| `testFramework.imagePullPolicy` | `test-framework` image pull policy. | `IfNotPresent` | +| `testFramework.securityContext` | `test-framework` securityContext | `{}` | +| `downloadDashboards.env` | Environment variables to be passed to the `download-dashboards` container | `{}` | +| `downloadDashboards.envFromSecret` | Name of a Kubernetes secret (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `""` | +| `downloadDashboards.resources` | Resources of `download-dashboards` container | `{}` | +| `downloadDashboardsImage.registry` | Curl docker image registry | `docker.io` | +| `downloadDashboardsImage.repository` | Curl docker image repository | `curlimages/curl` | +| `downloadDashboardsImage.tag` | Curl docker image tag | `7.73.0` | +| `downloadDashboardsImage.sha` | Curl docker image sha (optional) | `""` | +| `downloadDashboardsImage.pullPolicy` | Curl docker image pull policy | `IfNotPresent` | +| `namespaceOverride` | Override the deployment namespace | `""` (`Release.Namespace`) | +| `serviceMonitor.enabled` | Use servicemonitor from prometheus operator | `false` | +| `serviceMonitor.namespace` | Namespace this servicemonitor is installed in | | +| `serviceMonitor.interval` | How frequently Prometheus should scrape | `1m` | +| `serviceMonitor.path` | Path to scrape | `/metrics` | +| `serviceMonitor.scheme` | Scheme to use for metrics scraping | `http` | +| `serviceMonitor.tlsConfig` | TLS configuration block for the endpoint | `{}` | +| `serviceMonitor.labels` | Labels for the servicemonitor passed to Prometheus Operator | `{}` | +| `serviceMonitor.scrapeTimeout` | Timeout after which the scrape is ended | `30s` | +| `serviceMonitor.relabelings` | RelabelConfigs to apply to samples before scraping. | `[]` | +| `serviceMonitor.metricRelabelings` | MetricRelabelConfigs to apply to samples before ingestion. | `[]` | +| `revisionHistoryLimit` | Number of old ReplicaSets to retain | `10` | +| `imageRenderer.enabled` | Enable the image-renderer deployment & service | `false` | +| `imageRenderer.image.registry` | image-renderer Image registry | `docker.io` | +| `imageRenderer.image.repository` | image-renderer Image repository | `grafana/grafana-image-renderer` | +| `imageRenderer.image.tag` | image-renderer Image tag | `latest` | +| `imageRenderer.image.sha` | image-renderer Image sha (optional) | `""` | +| `imageRenderer.image.pullPolicy` | image-renderer ImagePullPolicy | `Always` | +| `imageRenderer.env` | extra env-vars for image-renderer | `{}` | +| `imageRenderer.envValueFrom` | Environment variables for image-renderer from alternate sources. See the API docs on [EnvVarSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core) for format details. Can be templated | `{}` | +| `imageRenderer.serviceAccountName` | image-renderer deployment serviceAccountName | `""` | +| `imageRenderer.securityContext` | image-renderer deployment securityContext | `{}` | +| `imageRenderer.podAnnotations ` | image-renderer image-renderer pod annotation | `{}` | +| `imageRenderer.hostAliases` | image-renderer deployment Host Aliases | `[]` | +| `imageRenderer.priorityClassName` | image-renderer deployment priority class | `''` | +| `imageRenderer.service.enabled` | Enable the image-renderer service | `true` | +| `imageRenderer.service.portName` | image-renderer service port name | `http` | +| `imageRenderer.service.port` | image-renderer port used by deployment | `8081` | +| `imageRenderer.service.targetPort` | image-renderer service port used by service | `8081` | +| `imageRenderer.appProtocol` | Adds the appProtocol field to the service | `` | +| `imageRenderer.grafanaSubPath` | Grafana sub path to use for image renderer callback url | `''` | +| `imageRenderer.podPortName` | name of the image-renderer port on the pod | `http` | +| `imageRenderer.revisionHistoryLimit` | number of image-renderer replica sets to keep | `10` | +| `imageRenderer.networkPolicy.limitIngress` | Enable a NetworkPolicy to limit inbound traffic from only the created grafana pods | `true` | +| `imageRenderer.networkPolicy.limitEgress` | Enable a NetworkPolicy to limit outbound traffic to only the created grafana pods | `false` | +| `imageRenderer.resources` | Set resource limits for image-renderer pods | `{}` | +| `imageRenderer.nodeSelector` | Node labels for pod assignment | `{}` | +| `imageRenderer.tolerations` | Toleration labels for pod assignment | `[]` | +| `imageRenderer.affinity` | Affinity settings for pod assignment | `{}` | +| `networkPolicy.enabled` | Enable creation of NetworkPolicy resources. | `false` | +| `networkPolicy.allowExternal` | Don't require client label for connections | `true` | +| `networkPolicy.explicitNamespacesSelector` | A Kubernetes LabelSelector to explicitly select namespaces from which traffic could be allowed | `{}` | +| `networkPolicy.ingress` | Enable the creation of an ingress network policy | `true` | +| `networkPolicy.egress.enabled` | Enable the creation of an egress network policy | `false` | +| `networkPolicy.egress.ports` | An array of ports to allow for the egress | `[]` | +| `enableKubeBackwardCompatibility` | Enable backward compatibility of kubernetes where pod's defintion version below 1.13 doesn't have the enableServiceLinks option | `false` | + +### Example ingress with path + +With grafana 6.3 and above + +```yaml +grafana.ini: + server: + domain: monitoring.example.com + root_url: "%(protocol)s://%(domain)s/grafana" + serve_from_sub_path: true +ingress: + enabled: true + hosts: + - "monitoring.example.com" + path: "/grafana" +``` + +### Example of extraVolumeMounts and extraVolumes + +Configure additional volumes with `extraVolumes` and volume mounts with `extraVolumeMounts`. + +Example for `extraVolumeMounts` and corresponding `extraVolumes`: + +```yaml +extraVolumeMounts: + - name: plugins + mountPath: /var/lib/grafana/plugins + subPath: configs/grafana/plugins + readOnly: false + - name: dashboards + mountPath: /var/lib/grafana/dashboards + hostPath: /usr/shared/grafana/dashboards + readOnly: false + +extraVolumes: + - name: plugins + existingClaim: existing-grafana-claim + - name: dashboards + hostPath: /usr/shared/grafana/dashboards +``` + +Volumes default to `emptyDir`. Set to `persistentVolumeClaim`, +`hostPath`, `csi`, or `configMap` for other types. For a +`persistentVolumeClaim`, specify an existing claim name with +`existingClaim`. + +## Import dashboards + +There are a few methods to import dashboards to Grafana. Below are some examples and explanations as to how to use each method: + +```yaml +dashboards: + default: + some-dashboard: + json: | + { + "annotations": + + ... + # Complete json file here + ... + + "title": "Some Dashboard", + "uid": "abcd1234", + "version": 1 + } + custom-dashboard: + # This is a path to a file inside the dashboards directory inside the chart directory + file: dashboards/custom-dashboard.json + prometheus-stats: + # Ref: https://grafana.com/dashboards/2 + gnetId: 2 + revision: 2 + datasource: Prometheus + loki-dashboard-quick-search: + gnetId: 12019 + revision: 2 + datasource: + - name: DS_PROMETHEUS + value: Prometheus + - name: DS_LOKI + value: Loki + local-dashboard: + url: https://raw.githubusercontent.com/user/repository/master/dashboards/dashboard.json +``` + +## BASE64 dashboards + +Dashboards could be stored on a server that does not return JSON directly and instead of it returns a Base64 encoded file (e.g. Gerrit) +A new parameter has been added to the url use case so if you specify a b64content value equals to true after the url entry a Base64 decoding is applied before save the file to disk. +If this entry is not set or is equals to false not decoding is applied to the file before saving it to disk. + +### Gerrit use case + +Gerrit API for download files has the following schema: where {project-name} and +{file-id} usually has '/' in their values and so they MUST be replaced by %2F so if project-name is user/repo, branch-id is master and file-id is equals to dir1/dir2/dashboard +the url value is + +## Sidecar for dashboards + +If the parameter `sidecar.dashboards.enabled` is set, a sidecar container is deployed in the grafana +pod. This container watches all configmaps (or secrets) in the cluster and filters out the ones with +a label as defined in `sidecar.dashboards.label`. The files defined in those configmaps are written +to a folder and accessed by grafana. Changes to the configmaps are monitored and the imported +dashboards are deleted/updated. + +A recommendation is to use one configmap per dashboard, as a reduction of multiple dashboards inside +one configmap is currently not properly mirrored in grafana. + +Example dashboard config: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: sample-grafana-dashboard + labels: + grafana_dashboard: "1" +data: + k8s-dashboard.json: |- + [...] +``` + +## Sidecar for datasources + +If the parameter `sidecar.datasources.enabled` is set, an init container is deployed in the grafana +pod. This container lists all secrets (or configmaps, though not recommended) in the cluster and +filters out the ones with a label as defined in `sidecar.datasources.label`. The files defined in +those secrets are written to a folder and accessed by grafana on startup. Using these yaml files, +the data sources in grafana can be imported. + +Should you aim for reloading datasources in Grafana each time the config is changed, set `sidecar.datasources.skipReload: false` and adjust `sidecar.datasources.reloadURL` to `http://..svc.cluster.local/api/admin/provisioning/datasources/reload`. + +Secrets are recommended over configmaps for this usecase because datasources usually contain private +data like usernames and passwords. Secrets are the more appropriate cluster resource to manage those. + +Example values to add a postgres datasource as a kubernetes secret: +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: grafana-datasources + labels: + grafana_datasource: 'true' # default value for: sidecar.datasources.label +stringData: + pg-db.yaml: |- + apiVersion: 1 + datasources: + - name: My pg db datasource + type: postgres + url: my-postgresql-db:5432 + user: db-readonly-user + secureJsonData: + password: 'SUperSEcretPa$$word' + jsonData: + database: my_datase + sslmode: 'disable' # disable/require/verify-ca/verify-full + maxOpenConns: 0 # Grafana v5.4+ + maxIdleConns: 2 # Grafana v5.4+ + connMaxLifetime: 14400 # Grafana v5.4+ + postgresVersion: 1000 # 903=9.3, 904=9.4, 905=9.5, 906=9.6, 1000=10 + timescaledb: false + # allow users to edit datasources from the UI. + editable: false +``` + +Example values to add a datasource adapted from [Grafana](http://docs.grafana.org/administration/provisioning/#example-datasource-config-file): + +```yaml +datasources: + datasources.yaml: + apiVersion: 1 + datasources: + # name of the datasource. Required + - name: Graphite + # datasource type. Required + type: graphite + # access mode. proxy or direct (Server or Browser in the UI). Required + access: proxy + # org id. will default to orgId 1 if not specified + orgId: 1 + # url + url: http://localhost:8080 + # database password, if used + password: + # database user, if used + user: + # database name, if used + database: + # enable/disable basic auth + basicAuth: + # basic auth username + basicAuthUser: + # basic auth password + basicAuthPassword: + # enable/disable with credentials headers + withCredentials: + # mark as default datasource. Max one per org + isDefault: + # fields that will be converted to json and stored in json_data + jsonData: + graphiteVersion: "1.1" + tlsAuth: true + tlsAuthWithCACert: true + # json object of data that will be encrypted. + secureJsonData: + tlsCACert: "..." + tlsClientCert: "..." + tlsClientKey: "..." + version: 1 + # allow users to edit datasources from the UI. + editable: false +``` + +## Sidecar for notifiers + +If the parameter `sidecar.notifiers.enabled` is set, an init container is deployed in the grafana +pod. This container lists all secrets (or configmaps, though not recommended) in the cluster and +filters out the ones with a label as defined in `sidecar.notifiers.label`. The files defined in +those secrets are written to a folder and accessed by grafana on startup. Using these yaml files, +the notification channels in grafana can be imported. The secrets must be created before +`helm install` so that the notifiers init container can list the secrets. + +Secrets are recommended over configmaps for this usecase because alert notification channels usually contain +private data like SMTP usernames and passwords. Secrets are the more appropriate cluster resource to manage those. + +Example datasource config adapted from [Grafana](https://grafana.com/docs/grafana/latest/administration/provisioning/#alert-notification-channels): + +```yaml +notifiers: + - name: notification-channel-1 + type: slack + uid: notifier1 + # either + org_id: 2 + # or + org_name: Main Org. + is_default: true + send_reminder: true + frequency: 1h + disable_resolve_message: false + # See `Supported Settings` section for settings supporter for each + # alert notification type. + settings: + recipient: 'XXX' + token: 'xoxb' + uploadImage: true + url: https://slack.com + +delete_notifiers: + - name: notification-channel-1 + uid: notifier1 + org_id: 2 + - name: notification-channel-2 + # default org_id: 1 +``` + +## Sidecar for alerting resources + +If the parameter `sidecar.alerts.enabled` is set, a sidecar container is deployed in the grafana +pod. This container watches all configmaps (or secrets) in the cluster (namespace defined by `sidecar.alerts.searchNamespace`) and filters out the ones with +a label as defined in `sidecar.alerts.label` (default is `grafana_alert`). The files defined in those configmaps are written +to a folder and accessed by grafana. Changes to the configmaps are monitored and the imported alerting resources are updated, however, deletions are a little more complicated (see below). + +This sidecar can be used to provision alert rules, contact points, notification policies, notification templates and mute timings as shown in [Grafana Documentation](https://grafana.com/docs/grafana/next/alerting/set-up/provision-alerting-resources/file-provisioning/). + +To fetch the alert config which will be provisioned, use the alert provisioning API ([Grafana Documentation](https://grafana.com/docs/grafana/next/developers/http_api/alerting_provisioning/)). +You can use either JSON or YAML format. + +Example config for an alert rule: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: sample-grafana-alert + labels: + grafana_alert: "1" +data: + k8s-alert.yml: |- + apiVersion: 1 + groups: + - orgId: 1 + name: k8s-alert + [...] +``` + +To delete provisioned alert rules is a two step process, you need to delete the configmap which defined the alert rule +and then create a configuration which deletes the alert rule. + +Example deletion configuration: +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: delete-sample-grafana-alert + namespace: monitoring + labels: + grafana_alert: "1" +data: + delete-k8s-alert.yml: |- + apiVersion: 1 + deleteRules: + - orgId: 1 + uid: 16624780-6564-45dc-825c-8bded4ad92d3 +``` + +## Statically provision alerting resources +If you don't need to change alerting resources (alert rules, contact points, notification policies and notification templates) regularly you could use the `alerting` config option instead of the sidecar option above. +This will grab the alerting config and apply it statically at build time for the helm file. + +There are two methods to statically provision alerting configuration in Grafana. Below are some examples and explanations as to how to use each method: + +```yaml +alerting: + team1-alert-rules.yaml: + file: alerting/team1/rules.yaml + team2-alert-rules.yaml: + file: alerting/team2/rules.yaml + team3-alert-rules.yaml: + file: alerting/team3/rules.yaml + notification-policies.yaml: + file: alerting/shared/notification-policies.yaml + notification-templates.yaml: + file: alerting/shared/notification-templates.yaml + contactpoints.yaml: + apiVersion: 1 + contactPoints: + - orgId: 1 + name: Slack channel + receivers: + - uid: default-receiver + type: slack + settings: + # Webhook URL to be filled in + url: "" + # We need to escape double curly braces for the tpl function. + text: '{{ `{{ template "default.message" . }}` }}' + title: '{{ `{{ template "default.title" . }}` }}' +``` + +The two possibilities for static alerting resource provisioning are: + +* Inlining the file contents as shown for contact points in the above example. +* Importing a file using a relative path starting from the chart root directory as shown for the alert rules in the above example. + +### Important notes on file provisioning + +* The format of the files is defined in the [Grafana documentation](https://grafana.com/docs/grafana/next/alerting/set-up/provision-alerting-resources/file-provisioning/) on file provisioning. +* The chart supports importing YAML and JSON files. +* The filename must be unique, otherwise one volume mount will overwrite the other. +* In case of inlining, double curly braces that arise from the Grafana configuration format and are not intended as templates for the chart must be escaped. +* The number of total files under `alerting:` is not limited. Each file will end up as a volume mount in the corresponding provisioning folder of the deployed Grafana instance. +* The file size for each import is limited by what the function `.Files.Get` can handle, which suffices for most cases. + +## How to serve Grafana with a path prefix (/grafana) + +In order to serve Grafana with a prefix (e.g., ), add the following to your values.yaml. + +```yaml +ingress: + enabled: true + annotations: + kubernetes.io/ingress.class: "nginx" + nginx.ingress.kubernetes.io/rewrite-target: /$1 + nginx.ingress.kubernetes.io/use-regex: "true" + + path: /grafana/?(.*) + hosts: + - k8s.example.dev + +grafana.ini: + server: + root_url: http://localhost:3000/grafana # this host can be localhost +``` + +## How to securely reference secrets in grafana.ini + +This example uses Grafana [file providers](https://grafana.com/docs/grafana/latest/administration/configuration/#file-provider) for secret values and the `extraSecretMounts` configuration flag (Additional grafana server secret mounts) to mount the secrets. + +In grafana.ini: + +```yaml +grafana.ini: + [auth.generic_oauth] + enabled = true + client_id = $__file{/etc/secrets/auth_generic_oauth/client_id} + client_secret = $__file{/etc/secrets/auth_generic_oauth/client_secret} +``` + +Existing secret, or created along with helm: + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: auth-generic-oauth-secret +type: Opaque +stringData: + client_id: + client_secret: +``` + +Include in the `extraSecretMounts` configuration flag: + +```yaml +- extraSecretMounts: + - name: auth-generic-oauth-secret-mount + secretName: auth-generic-oauth-secret + defaultMode: 0440 + mountPath: /etc/secrets/auth_generic_oauth + readOnly: true +``` + +### extraSecretMounts using a Container Storage Interface (CSI) provider + +This example uses a CSI driver e.g. retrieving secrets using [Azure Key Vault Provider](https://github.com/Azure/secrets-store-csi-driver-provider-azure) + +```yaml +- extraSecretMounts: + - name: secrets-store-inline + mountPath: /run/secrets + readOnly: true + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: "my-provider" + nodePublishSecretRef: + name: akv-creds +``` + +## Image Renderer Plug-In + +This chart supports enabling [remote image rendering](https://github.com/grafana/grafana-image-renderer/blob/master/README.md#run-in-docker) + +```yaml +imageRenderer: + enabled: true +``` + +### Image Renderer NetworkPolicy + +By default the image-renderer pods will have a network policy which only allows ingress traffic from the created grafana instance + +### High Availability for unified alerting + +If you want to run Grafana in a high availability cluster you need to enable +the headless service by setting `headlessService: true` in your `values.yaml` +file. + +As next step you have to setup the `grafana.ini` in your `values.yaml` in a way +that it will make use of the headless service to obtain all the IPs of the +cluster. You should replace ``{{ Name }}`` with the name of your helm deployment. + +```yaml +grafana.ini: + ... + unified_alerting: + enabled: true + ha_peers: {{ Name }}-headless:9094 + ha_listen_address: ${POD_IP}:9094 + ha_advertise_address: ${POD_IP}:9094 + + alerting: + enabled: false +``` diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/dashboards/custom-dashboard.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/dashboards/custom-dashboard.json new file mode 100644 index 0000000000..9e26dfeeb6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/dashboards/custom-dashboard.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/NOTES.txt b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/NOTES.txt new file mode 100644 index 0000000000..d86419fe23 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/NOTES.txt @@ -0,0 +1,55 @@ +1. Get your '{{ .Values.adminUser }}' user password by running: + + kubectl get secret --namespace {{ include "grafana.namespace" . }} {{ .Values.admin.existingSecret | default (include "grafana.fullname" .) }} -o jsonpath="{.data.{{ .Values.admin.passwordKey | default "admin-password" }}}" | base64 --decode ; echo + + +2. The Grafana server can be accessed via port {{ .Values.service.port }} on the following DNS name from within your cluster: + + {{ include "grafana.fullname" . }}.{{ include "grafana.namespace" . }}.svc.cluster.local +{{ if .Values.ingress.enabled }} + If you bind grafana to 80, please update values in values.yaml and reinstall: + ``` + securityContext: + runAsUser: 0 + runAsGroup: 0 + fsGroup: 0 + + command: + - "setcap" + - "'cap_net_bind_service=+ep'" + - "/usr/sbin/grafana-server &&" + - "sh" + - "/run.sh" + ``` + Details refer to https://grafana.com/docs/installation/configuration/#http-port. + Or grafana would always crash. + + From outside the cluster, the server URL(s) are: + {{- range .Values.ingress.hosts }} + http://{{ . }} + {{- end }} +{{- else }} + Get the Grafana URL to visit by running these commands in the same shell: + {{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ include "grafana.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "grafana.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ include "grafana.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT + {{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc --namespace {{ include "grafana.namespace" . }} -w {{ include "grafana.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ include "grafana.namespace" . }} {{ include "grafana.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + http://$SERVICE_IP:{{ .Values.service.port -}} + {{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ include "grafana.namespace" . }} -l "app.kubernetes.io/name={{ include "grafana.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + kubectl --namespace {{ include "grafana.namespace" . }} port-forward $POD_NAME 3000 + {{- end }} +{{- end }} + +3. Login with the password from step 1 and the username: {{ .Values.adminUser }} + +{{- if not .Values.persistence.enabled }} +################################################################################# +###### WARNING: Persistence is disabled!!! You will lose your data when ##### +###### the Grafana pod is terminated. ##### +################################################################################# +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/_config.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/_config.tpl new file mode 100644 index 0000000000..19df19cd2a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/_config.tpl @@ -0,0 +1,171 @@ +{{/* + Generate config map data + */}} +{{- define "grafana.configData" -}} +{{ include "grafana.assertNoLeakedSecrets" . }} +{{- $files := .Files }} +{{- $root := . -}} +{{- with .Values.plugins }} +plugins: {{ join "," . }} +{{- end }} +grafana.ini: | +{{- range $elem, $elemVal := index .Values "grafana.ini" }} + {{- if not (kindIs "map" $elemVal) }} + {{- if kindIs "invalid" $elemVal }} + {{ $elem }} = + {{- else if kindIs "string" $elemVal }} + {{ $elem }} = {{ tpl $elemVal $ }} + {{- else }} + {{ $elem }} = {{ $elemVal }} + {{- end }} + {{- end }} +{{- end }} +{{- range $key, $value := index .Values "grafana.ini" }} + {{- if kindIs "map" $value }} + [{{ $key }}] + {{- range $elem, $elemVal := $value }} + {{- if kindIs "invalid" $elemVal }} + {{ $elem }} = + {{- else if kindIs "string" $elemVal }} + {{ $elem }} = {{ tpl $elemVal $ }} + {{- else }} + {{ $elem }} = {{ $elemVal }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} + +{{- range $key, $value := .Values.datasources }} +{{- if not (hasKey $value "secret") }} +{{ $key }}: | + {{- tpl (toYaml $value | nindent 2) $root }} +{{- end }} +{{- end }} + +{{- range $key, $value := .Values.notifiers }} +{{- if not (hasKey $value "secret") }} +{{ $key }}: | + {{- toYaml $value | nindent 2 }} +{{- end }} +{{- end }} + +{{- range $key, $value := .Values.alerting }} +{{- if (hasKey $value "file") }} +{{ $key }}: +{{- toYaml ( $files.Get $value.file ) | nindent 2 }} +{{- else if (or (hasKey $value "secret") (hasKey $value "secretFile"))}} +{{/* will be stored inside secret generated by "configSecret.yaml"*/}} +{{- else }} +{{ $key }}: | + {{- tpl (toYaml $value | nindent 2) $root }} +{{- end }} +{{- end }} + +{{- range $key, $value := .Values.dashboardProviders }} +{{ $key }}: | + {{- toYaml $value | nindent 2 }} +{{- end }} + +{{- if .Values.dashboards }} +download_dashboards.sh: | + #!/usr/bin/env sh + set -euf + {{- if .Values.dashboardProviders }} + {{- range $key, $value := .Values.dashboardProviders }} + {{- range $value.providers }} + mkdir -p {{ .options.path }} + {{- end }} + {{- end }} + {{- end }} +{{ $dashboardProviders := .Values.dashboardProviders }} +{{- range $provider, $dashboards := .Values.dashboards }} + {{- range $key, $value := $dashboards }} + {{- if (or (hasKey $value "gnetId") (hasKey $value "url")) }} + curl -skf \ + --connect-timeout 60 \ + --max-time 60 \ + {{- if not $value.b64content }} + {{- if not $value.acceptHeader }} + -H "Accept: application/json" \ + {{- else }} + -H "Accept: {{ $value.acceptHeader }}" \ + {{- end }} + {{- if $value.token }} + -H "Authorization: token {{ $value.token }}" \ + {{- end }} + {{- if $value.bearerToken }} + -H "Authorization: Bearer {{ $value.bearerToken }}" \ + {{- end }} + {{- if $value.basic }} + -H "Authorization: Basic {{ $value.basic }}" \ + {{- end }} + {{- if $value.gitlabToken }} + -H "PRIVATE-TOKEN: {{ $value.gitlabToken }}" \ + {{- end }} + -H "Content-Type: application/json;charset=UTF-8" \ + {{- end }} + {{- $dpPath := "" -}} + {{- range $kd := (index $dashboardProviders "dashboardproviders.yaml").providers }} + {{- if eq $kd.name $provider }} + {{- $dpPath = $kd.options.path }} + {{- end }} + {{- end }} + {{- if $value.url }} + "{{ $value.url }}" \ + {{- else }} + "https://grafana.com/api/dashboards/{{ $value.gnetId }}/revisions/{{- if $value.revision -}}{{ $value.revision }}{{- else -}}1{{- end -}}/download" \ + {{- end }} + {{- if $value.datasource }} + {{- if kindIs "string" $value.datasource }} + | sed '/-- .* --/! s/"datasource":.*,/"datasource": "{{ $value.datasource }}",/g' \ + {{- end }} + {{- if kindIs "slice" $value.datasource }} + {{- range $value.datasource }} + | sed '/-- .* --/! s/${{"{"}}{{ .name }}}/{{ .value }}/g' \ + {{- end }} + {{- end }} + {{- end }} + {{- if $value.b64content }} + | base64 -d \ + {{- end }} + > "{{- if $dpPath -}}{{ $dpPath }}{{- else -}}/var/lib/grafana/dashboards/{{ $provider }}{{- end -}}/{{ $key }}.json" + {{ end }} + {{- end }} +{{- end }} +{{- end }} +{{- end -}} + +{{/* + Generate dashboard json config map data + */}} +{{- define "grafana.configDashboardProviderData" -}} +provider.yaml: |- + apiVersion: 1 + providers: + - name: '{{ .Values.sidecar.dashboards.provider.name }}' + orgId: {{ .Values.sidecar.dashboards.provider.orgid }} + {{- if not .Values.sidecar.dashboards.provider.foldersFromFilesStructure }} + folder: '{{ .Values.sidecar.dashboards.provider.folder }}' + {{- end }} + type: {{ .Values.sidecar.dashboards.provider.type }} + disableDeletion: {{ .Values.sidecar.dashboards.provider.disableDelete }} + allowUiUpdates: {{ .Values.sidecar.dashboards.provider.allowUiUpdates }} + updateIntervalSeconds: {{ .Values.sidecar.dashboards.provider.updateIntervalSeconds | default 30 }} + options: + foldersFromFilesStructure: {{ .Values.sidecar.dashboards.provider.foldersFromFilesStructure }} + path: {{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }} +{{- end -}} + +{{- define "grafana.secretsData" -}} +{{- if and (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) }} +admin-user: {{ .Values.adminUser | b64enc | quote }} +{{- if .Values.adminPassword }} +admin-password: {{ .Values.adminPassword | b64enc | quote }} +{{- else }} +admin-password: {{ include "grafana.password" . }} +{{- end }} +{{- end }} +{{- if not .Values.ldap.existingSecret }} +ldap-toml: {{ tpl .Values.ldap.config $ | b64enc | quote }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/_helpers.tpl new file mode 100644 index 0000000000..68d2d815d8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/_helpers.tpl @@ -0,0 +1,305 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "grafana.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "grafana.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "grafana.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create the name of the service account +*/}} +{{- define "grafana.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "grafana.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{- define "grafana.serviceAccountNameTest" -}} +{{- if .Values.serviceAccount.create }} +{{- default (print (include "grafana.fullname" .) "-test") .Values.serviceAccount.nameTest }} +{{- else }} +{{- default "default" .Values.serviceAccount.nameTest }} +{{- end }} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "grafana.namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "grafana.labels" -}} +helm.sh/chart: {{ include "grafana.chart" . }} +{{ include "grafana.selectorLabels" . }} +{{- if or .Chart.AppVersion .Values.image.tag }} +app.kubernetes.io/version: {{ mustRegexReplaceAllLiteral "@sha.*" .Values.image.tag "" | default .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- with .Values.extraLabels }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "grafana.selectorLabels" -}} +app.kubernetes.io/name: {{ include "grafana.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "grafana.imageRenderer.labels" -}} +helm.sh/chart: {{ include "grafana.chart" . }} +{{ include "grafana.imageRenderer.selectorLabels" . }} +{{- if or .Chart.AppVersion .Values.image.tag }} +app.kubernetes.io/version: {{ mustRegexReplaceAllLiteral "@sha.*" .Values.image.tag "" | default .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels ImageRenderer +*/}} +{{- define "grafana.imageRenderer.selectorLabels" -}} +app.kubernetes.io/name: {{ include "grafana.name" . }}-image-renderer +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Looks if there's an existing secret and reuse its password. If not it generates +new password and use it. +*/}} +{{- define "grafana.password" -}} +{{- $secret := (lookup "v1" "Secret" (include "grafana.namespace" .) (include "grafana.fullname" .) ) }} +{{- if $secret }} +{{- index $secret "data" "admin-password" }} +{{- else }} +{{- (randAlphaNum 40) | b64enc | quote }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for rbac. +*/}} +{{- define "grafana.rbac.apiVersion" -}} +{{- if $.Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1" }} +{{- print "rbac.authorization.k8s.io/v1" }} +{{- else }} +{{- print "rbac.authorization.k8s.io/v1beta1" }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "grafana.ingress.apiVersion" -}} +{{- if and ($.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" .Capabilities.KubeVersion.Version) }} +{{- print "networking.k8s.io/v1" }} +{{- else if $.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }} +{{- print "networking.k8s.io/v1beta1" }} +{{- else }} +{{- print "extensions/v1beta1" }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for Horizontal Pod Autoscaler. +*/}} +{{- define "grafana.hpa.apiVersion" -}} +{{- if .Capabilities.APIVersions.Has "autoscaling/v2" }} +{{- print "autoscaling/v2" }} +{{- else }} +{{- print "autoscaling/v2beta2" }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for podDisruptionBudget. +*/}} +{{- define "grafana.podDisruptionBudget.apiVersion" -}} +{{- if $.Values.podDisruptionBudget.apiVersion }} +{{- print $.Values.podDisruptionBudget.apiVersion }} +{{- else if $.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget" }} +{{- print "policy/v1" }} +{{- else }} +{{- print "policy/v1beta1" }} +{{- end }} +{{- end }} + +{{/* +Return if ingress is stable. +*/}} +{{- define "grafana.ingress.isStable" -}} +{{- eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1" }} +{{- end }} + +{{/* +Return if ingress supports ingressClassName. +*/}} +{{- define "grafana.ingress.supportsIngressClassName" -}} +{{- or (eq (include "grafana.ingress.isStable" .) "true") (and (eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) }} +{{- end }} + +{{/* +Return if ingress supports pathType. +*/}} +{{- define "grafana.ingress.supportsPathType" -}} +{{- or (eq (include "grafana.ingress.isStable" .) "true") (and (eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) }} +{{- end }} + +{{/* +Formats imagePullSecrets. Input is (dict "root" . "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "grafana.imagePullSecrets" -}} +{{- $root := .root }} +{{- range (concat .root.Values.global.imagePullSecrets .imagePullSecrets) }} +{{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml (dict "name" (tpl .name $root)) | trim }} +{{- else }} +- name: {{ tpl . $root }} +{{- end }} +{{- end }} +{{- end }} + + +{{/* + Checks whether or not the configSecret secret has to be created + */}} +{{- define "grafana.shouldCreateConfigSecret" -}} +{{- $secretFound := false -}} +{{- range $key, $value := .Values.datasources }} + {{- if hasKey $value "secret" }} + {{- $secretFound = true}} + {{- end }} +{{- end }} +{{- range $key, $value := .Values.notifiers }} + {{- if hasKey $value "secret" }} + {{- $secretFound = true}} + {{- end }} +{{- end }} +{{- range $key, $value := .Values.alerting }} + {{- if (or (hasKey $value "secret") (hasKey $value "secretFile")) }} + {{- $secretFound = true}} + {{- end }} +{{- end }} +{{- $secretFound}} +{{- end -}} + +{{/* + Checks whether the user is attempting to store secrets in plaintext + in the grafana.ini configmap +*/}} +{{/* grafana.assertNoLeakedSecrets checks for sensitive keys in values */}} +{{- define "grafana.assertNoLeakedSecrets" -}} + {{- $sensitiveKeysYaml := ` +sensitiveKeys: +- path: ["database", "password"] +- path: ["smtp", "password"] +- path: ["security", "secret_key"] +- path: ["security", "admin_password"] +- path: ["auth.basic", "password"] +- path: ["auth.ldap", "bind_password"] +- path: ["auth.google", "client_secret"] +- path: ["auth.github", "client_secret"] +- path: ["auth.gitlab", "client_secret"] +- path: ["auth.generic_oauth", "client_secret"] +- path: ["auth.okta", "client_secret"] +- path: ["auth.azuread", "client_secret"] +- path: ["auth.grafana_com", "client_secret"] +- path: ["auth.grafananet", "client_secret"] +- path: ["azure", "user_identity_client_secret"] +- path: ["unified_alerting", "ha_redis_password"] +- path: ["metrics", "basic_auth_password"] +- path: ["external_image_storage.s3", "secret_key"] +- path: ["external_image_storage.webdav", "password"] +- path: ["external_image_storage.azure_blob", "account_key"] +` | fromYaml -}} + {{- if $.Values.assertNoLeakedSecrets -}} + {{- $grafanaIni := index .Values "grafana.ini" -}} + {{- range $_, $secret := $sensitiveKeysYaml.sensitiveKeys -}} + {{- $currentMap := $grafanaIni -}} + {{- $shouldContinue := true -}} + {{- range $index, $elem := $secret.path -}} + {{- if and $shouldContinue (hasKey $currentMap $elem) -}} + {{- if eq (len $secret.path) (add1 $index) -}} + {{- if not (regexMatch "\\$(?:__(?:env|file|vault))?{[^}]+}" (index $currentMap $elem)) -}} + {{- fail (printf "Sensitive key '%s' should not be defined explicitly in values. Use variable expansion instead. You can disable this client-side validation by changing the value of assertNoLeakedSecrets." (join "." $secret.path)) -}} + {{- end -}} + {{- else -}} + {{- $currentMap = index $currentMap $elem -}} + {{- end -}} + {{- else -}} + {{- $shouldContinue = false -}} + {{- end -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/_pod.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/_pod.tpl new file mode 100644 index 0000000000..2ebf7d5f10 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/_pod.tpl @@ -0,0 +1,1296 @@ +{{- define "grafana.pod" -}} +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- $root := . -}} +{{- with .Values.schedulerName }} +schedulerName: "{{ . }}" +{{- end }} +serviceAccountName: {{ include "grafana.serviceAccountName" . }} +automountServiceAccountToken: {{ .Values.automountServiceAccountToken }} +{{- with .Values.securityContext }} +securityContext: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.hostAliases }} +hostAliases: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- if .Values.dnsPolicy }} +dnsPolicy: {{ .Values.dnsPolicy }} +{{- end }} +{{- with .Values.dnsConfig }} +dnsConfig: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.priorityClassName }} +priorityClassName: {{ . }} +{{- end }} +{{- if ( or .Values.persistence.enabled .Values.dashboards .Values.extraInitContainers (and .Values.sidecar.alerts.enabled .Values.sidecar.alerts.initAlerts) (and .Values.sidecar.datasources.enabled .Values.sidecar.datasources.initDatasources) (and .Values.sidecar.notifiers.enabled .Values.sidecar.notifiers.initNotifiers)) }} +initContainers: +{{- end }} +{{- if ( and .Values.persistence.enabled .Values.initChownData.enabled ) }} + - name: init-chown-data + {{- $registry := include "system_default_registry" . | default .Values.initChownData.image.registry -}} + {{- if .Values.initChownData.image.sha }} + image: "{{ $registry }}{{ .Values.initChownData.image.repository }}:{{ .Values.initChownData.image.tag }}@sha256:{{ .Values.initChownData.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.initChownData.image.repository }}:{{ .Values.initChownData.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.initChownData.image.pullPolicy }} + {{- with .Values.initChownData.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + command: + - chown + - -R + - {{ .Values.securityContext.runAsUser }}:{{ .Values.securityContext.runAsGroup }} + - /var/lib/grafana + {{- with .Values.initChownData.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: storage + mountPath: "/var/lib/grafana" + {{- with .Values.persistence.subPath }} + subPath: {{ tpl . $root }} + {{- end }} +{{- end }} +{{- if .Values.dashboards }} + - name: download-dashboards + {{- $registry := include "system_default_registry" . | default .Values.downloadDashboardsImage.registry -}} + {{- if .Values.downloadDashboardsImage.sha }} + image: "{{ $registry }}{{ .Values.downloadDashboardsImage.repository }}:{{ .Values.downloadDashboardsImage.tag }}@sha256:{{ .Values.downloadDashboardsImage.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.downloadDashboardsImage.repository }}:{{ .Values.downloadDashboardsImage.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.downloadDashboardsImage.pullPolicy }} + command: ["/bin/sh"] + args: [ "-c", "mkdir -p /var/lib/grafana/dashboards/default && /bin/sh -x /etc/grafana/download_dashboards.sh" ] + {{- with .Values.downloadDashboards.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + env: + {{- range $key, $value := .Values.downloadDashboards.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- range $key, $value := .Values.downloadDashboards.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- with .Values.downloadDashboards.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.downloadDashboards.envFromSecret }} + envFrom: + - secretRef: + name: {{ tpl . $root }} + {{- end }} + volumeMounts: + - name: config + mountPath: "/etc/grafana/download_dashboards.sh" + subPath: download_dashboards.sh + - name: storage + mountPath: "/var/lib/grafana" + {{- with .Values.persistence.subPath }} + subPath: {{ tpl . $root }} + {{- end }} + {{- range .Values.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + {{- end }} +{{- end }} +{{- if and .Values.sidecar.alerts.enabled .Values.sidecar.alerts.initAlerts }} + - name: {{ include "grafana.name" . }}-init-sc-alerts + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.alerts.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.alerts.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: "LIST" + - name: LABEL + value: "{{ .Values.sidecar.alerts.label }}" + {{- with .Values.sidecar.alerts.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/alerting" + - name: RESOURCE + value: {{ quote .Values.sidecar.alerts.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.alerts.searchNamespace }} + - name: NAMESPACE + value: {{ . | join "," | quote }} + {{- end }} + {{- with .Values.sidecar.alerts.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: {{ quote . }} + {{- end }} + {{- with .Values.sidecar.alerts.script }} + - name: SCRIPT + value: {{ quote . }} + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-alerts-volume + mountPath: "/etc/grafana/provisioning/alerting" + {{- with .Values.sidecar.alerts.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end }} +{{- if and .Values.sidecar.datasources.enabled .Values.sidecar.datasources.initDatasources }} + - name: {{ include "grafana.name" . }}-init-sc-datasources + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.datasources.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.datasources.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: "LIST" + - name: LABEL + value: "{{ .Values.sidecar.datasources.label }}" + {{- with .Values.sidecar.datasources.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/datasources" + - name: RESOURCE + value: {{ quote .Values.sidecar.datasources.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- if .Values.sidecar.datasources.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (.Values.sidecar.datasources.searchNamespace | join ",") . }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" +{{- end }} +{{- if and .Values.sidecar.notifiers.enabled .Values.sidecar.notifiers.initNotifiers }} + - name: {{ include "grafana.name" . }}-init-sc-notifiers + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.notifiers.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.notifiers.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: LIST + - name: LABEL + value: "{{ .Values.sidecar.notifiers.label }}" + {{- with .Values.sidecar.notifiers.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/notifiers" + - name: RESOURCE + value: {{ quote .Values.sidecar.notifiers.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.notifiers.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" +{{- end}} +{{- with .Values.extraInitContainers }} + {{- tpl (toYaml .) $root | nindent 2 }} +{{- end }} +{{- if or .Values.image.pullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "grafana.imagePullSecrets" (dict "root" $root "imagePullSecrets" .Values.image.pullSecrets) | nindent 2 }} +{{- end }} +{{- if not .Values.enableKubeBackwardCompatibility }} +enableServiceLinks: {{ .Values.enableServiceLinks }} +{{- end }} +containers: +{{- if and .Values.sidecar.alerts.enabled (not .Values.sidecar.alerts.initAlerts) }} + - name: {{ include "grafana.name" . }}-sc-alerts + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.alerts.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.alerts.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.alerts.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.alerts.label }}" + {{- with .Values.sidecar.alerts.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/alerting" + - name: RESOURCE + value: {{ quote .Values.sidecar.alerts.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.alerts.searchNamespace }} + - name: NAMESPACE + value: {{ . | join "," | quote }} + {{- end }} + {{- with .Values.sidecar.alerts.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: {{ quote . }} + {{- end }} + {{- with .Values.sidecar.alerts.script }} + - name: SCRIPT + value: {{ quote . }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.alerts.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.alerts.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.alerts.watchServerTimeout }} + {{- if ne .Values.sidecar.alerts.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.alerts.watchServerTimeout with .Values.sidecar.alerts.watchMethod %s" .Values.sidecar.alerts.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.alerts.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.alerts.watchClientTimeout }} + {{- if ne .Values.sidecar.alerts.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.alerts.watchClientTimeout with .Values.sidecar.alerts.watchMethod %s" .Values.sidecar.alerts.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.alerts.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-alerts-volume + mountPath: "/etc/grafana/provisioning/alerting" + {{- with .Values.sidecar.alerts.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end}} +{{- if .Values.sidecar.dashboards.enabled }} + - name: {{ include "grafana.name" . }}-sc-dashboard + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.dashboards.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- range $key, $value := .Values.sidecar.datasources.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- if .Values.sidecar.dashboards.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.dashboards.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.dashboards.label }}" + {{- with .Values.sidecar.dashboards.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.dashboards.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.dashboards.logLevel }} + {{- end }} + - name: FOLDER + value: "{{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }}" + - name: RESOURCE + value: {{ quote .Values.sidecar.dashboards.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.dashboards.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.dashboards.folderAnnotation }} + - name: FOLDER_ANNOTATION + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.dashboards.script }} + - name: SCRIPT + value: "{{ . }}" + {{- end }} + {{- if not .Values.sidecar.dashboards.skipReload }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + - name: REQ_URL + value: {{ .Values.sidecar.dashboards.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.dashboards.watchServerTimeout }} + {{- if ne .Values.sidecar.dashboards.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.dashboards.watchServerTimeout with .Values.sidecar.dashboards.watchMethod %s" .Values.sidecar.dashboards.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.dashboards.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.dashboards.watchClientTimeout }} + {{- if ne .Values.sidecar.dashboards.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.dashboards.watchClientTimeout with .Values.sidecar.dashboards.watchMethod %s" .Values.sidecar.dashboards.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: {{ .Values.sidecar.dashboards.watchClientTimeout | quote }} + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-dashboard-volume + mountPath: {{ .Values.sidecar.dashboards.folder | quote }} + {{- with .Values.sidecar.dashboards.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end}} +{{- if and .Values.sidecar.datasources.enabled (not .Values.sidecar.datasources.initDatasources) }} + - name: {{ include "grafana.name" . }}-sc-datasources + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.datasources.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.datasources.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.datasources.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.datasources.label }}" + {{- with .Values.sidecar.datasources.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/datasources" + - name: RESOURCE + value: {{ quote .Values.sidecar.datasources.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.datasources.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- if .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ .Values.sidecar.skipTlsVerify }}" + {{- end }} + {{- if .Values.sidecar.datasources.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.datasources.script }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.datasources.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.datasources.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.datasources.watchServerTimeout }} + {{- if ne .Values.sidecar.datasources.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.datasources.watchServerTimeout with .Values.sidecar.datasources.watchMethod %s" .Values.sidecar.datasources.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.datasources.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.datasources.watchClientTimeout }} + {{- if ne .Values.sidecar.datasources.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.datasources.watchClientTimeout with .Values.sidecar.datasources.watchMethod %s" .Values.sidecar.datasources.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.datasources.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" +{{- end}} +{{- if .Values.sidecar.notifiers.enabled }} + - name: {{ include "grafana.name" . }}-sc-notifiers + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.notifiers.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.notifiers.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.notifiers.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.notifiers.label }}" + {{- with .Values.sidecar.notifiers.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/notifiers" + - name: RESOURCE + value: {{ quote .Values.sidecar.notifiers.resource }} + {{- if .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ .Values.sidecar.enableUniqueFilenames }}" + {{- end }} + {{- with .Values.sidecar.notifiers.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- if .Values.sidecar.notifiers.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.notifiers.script }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.notifiers.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.notifiers.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.notifiers.watchServerTimeout }} + {{- if ne .Values.sidecar.notifiers.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.notifiers.watchServerTimeout with .Values.sidecar.notifiers.watchMethod %s" .Values.sidecar.notifiers.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.notifiers.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.notifiers.watchClientTimeout }} + {{- if ne .Values.sidecar.notifiers.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.notifiers.watchClientTimeout with .Values.sidecar.notifiers.watchMethod %s" .Values.sidecar.notifiers.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.notifiers.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" +{{- end}} +{{- if .Values.sidecar.plugins.enabled }} + - name: {{ include "grafana.name" . }}-sc-plugins + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.plugins.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.plugins.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.plugins.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.plugins.label }}" + {{- if .Values.sidecar.plugins.labelValue }} + - name: LABEL_VALUE + value: {{ quote .Values.sidecar.plugins.labelValue }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.plugins.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.plugins.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/plugins" + - name: RESOURCE + value: {{ quote .Values.sidecar.plugins.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.plugins.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.plugins.script }} + - name: SCRIPT + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.plugins.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.plugins.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.plugins.watchServerTimeout }} + {{- if ne .Values.sidecar.plugins.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.plugins.watchServerTimeout with .Values.sidecar.plugins.watchMethod %s" .Values.sidecar.plugins.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.plugins.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.plugins.watchClientTimeout }} + {{- if ne .Values.sidecar.plugins.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.plugins.watchClientTimeout with .Values.sidecar.plugins.watchMethod %s" .Values.sidecar.plugins.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.plugins.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-plugins-volume + mountPath: "/etc/grafana/provisioning/plugins" +{{- end}} + - name: {{ .Chart.Name }} + {{- $registry := include "system_default_registry" . | default .Values.image.registry -}} + {{- if .Values.image.sha }} + image: "{{ $registry }}{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.command }} + command: + {{- range .Values.command }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- if .Values.args }} + args: + {{- range .Values.args }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- with .Values.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: config + mountPath: "/etc/grafana/grafana.ini" + subPath: grafana.ini + {{- if .Values.ldap.enabled }} + - name: ldap + mountPath: "/etc/grafana/ldap.toml" + subPath: ldap.toml + {{- end }} + {{- range .Values.extraConfigmapMounts }} + - name: {{ tpl .name $root }} + mountPath: {{ tpl .mountPath $root }} + subPath: {{ tpl (.subPath | default "") $root }} + readOnly: {{ .readOnly }} + {{- end }} + - name: storage + mountPath: "/var/lib/grafana" + {{- with .Values.persistence.subPath }} + subPath: {{ tpl . $root }} + {{- end }} + {{- with .Values.dashboards }} + {{- range $provider, $dashboards := . }} + {{- range $key, $value := $dashboards }} + {{- if (or (hasKey $value "json") (hasKey $value "file")) }} + - name: dashboards-{{ $provider }} + mountPath: "/var/lib/grafana/dashboards/{{ $provider }}/{{ $key }}.json" + subPath: "{{ $key }}.json" + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.dashboardsConfigMaps }} + {{- range (keys . | sortAlpha) }} + - name: dashboards-{{ . }} + mountPath: "/var/lib/grafana/dashboards/{{ . }}" + {{- end }} + {{- end }} + {{- with .Values.datasources }} + {{- $datasources := . }} + {{- range (keys . | sortAlpha) }} + {{- if (or (hasKey (index $datasources .) "secret")) }} {{/*check if current datasource should be handeled as secret */}} + - name: config-secret + mountPath: "/etc/grafana/provisioning/datasources/{{ . }}" + subPath: {{ . | quote }} + {{- else }} + - name: config + mountPath: "/etc/grafana/provisioning/datasources/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.notifiers }} + {{- $notifiers := . }} + {{- range (keys . | sortAlpha) }} + {{- if (or (hasKey (index $notifiers .) "secret")) }} {{/*check if current notifier should be handeled as secret */}} + - name: config-secret + mountPath: "/etc/grafana/provisioning/notifiers/{{ . }}" + subPath: {{ . | quote }} + {{- else }} + - name: config + mountPath: "/etc/grafana/provisioning/notifiers/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.alerting }} + {{- $alertingmap := .}} + {{- range (keys . | sortAlpha) }} + {{- if (or (hasKey (index $.Values.alerting .) "secret") (hasKey (index $.Values.alerting .) "secretFile")) }} {{/*check if current alerting entry should be handeled as secret */}} + - name: config-secret + mountPath: "/etc/grafana/provisioning/alerting/{{ . }}" + subPath: {{ . | quote }} + {{- else }} + - name: config + mountPath: "/etc/grafana/provisioning/alerting/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.dashboardProviders }} + {{- range (keys . | sortAlpha) }} + - name: config + mountPath: "/etc/grafana/provisioning/dashboards/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- with .Values.sidecar.alerts.enabled }} + - name: sc-alerts-volume + mountPath: "/etc/grafana/provisioning/alerting" + {{- end}} + {{- if .Values.sidecar.dashboards.enabled }} + - name: sc-dashboard-volume + mountPath: {{ .Values.sidecar.dashboards.folder | quote }} + {{- if .Values.sidecar.dashboards.SCProvider }} + - name: sc-dashboard-provider + mountPath: "/etc/grafana/provisioning/dashboards/sc-dashboardproviders.yaml" + subPath: provider.yaml + {{- end}} + {{- end}} + {{- if .Values.sidecar.datasources.enabled }} + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" + {{- end}} + {{- if .Values.sidecar.plugins.enabled }} + - name: sc-plugins-volume + mountPath: "/etc/grafana/provisioning/plugins" + {{- end}} + {{- if .Values.sidecar.notifiers.enabled }} + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" + {{- end}} + {{- range .Values.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + subPath: {{ .subPath | default "" }} + {{- end }} + {{- range .Values.extraVolumeMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath | default "" }} + readOnly: {{ .readOnly }} + {{- end }} + {{- range .Values.extraEmptyDirMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + ports: + - name: {{ .Values.podPortName }} + containerPort: {{ .Values.service.targetPort }} + protocol: TCP + - name: {{ .Values.gossipPortName }}-tcp + containerPort: 9094 + protocol: TCP + - name: {{ .Values.gossipPortName }}-udp + containerPort: 9094 + protocol: UDP + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: GF_SECURITY_ADMIN_USER + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if .Values.plugins }} + - name: GF_INSTALL_PLUGINS + valueFrom: + configMapKeyRef: + name: {{ include "grafana.fullname" . }} + key: plugins + {{- end }} + {{- if .Values.smtp.existingSecret }} + - name: GF_SMTP_USER + valueFrom: + secretKeyRef: + name: {{ .Values.smtp.existingSecret }} + key: {{ .Values.smtp.userKey | default "user" }} + - name: GF_SMTP_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.smtp.existingSecret }} + key: {{ .Values.smtp.passwordKey | default "password" }} + {{- end }} + {{- if .Values.imageRenderer.enabled }} + - name: GF_RENDERING_SERVER_URL + value: http://{{ include "grafana.fullname" . }}-image-renderer.{{ include "grafana.namespace" . }}:{{ .Values.imageRenderer.service.port }}/render + - name: GF_RENDERING_CALLBACK_URL + value: {{ .Values.imageRenderer.grafanaProtocol }}://{{ include "grafana.fullname" . }}.{{ include "grafana.namespace" . }}:{{ .Values.service.port }}/{{ .Values.imageRenderer.grafanaSubPath }} + {{- end }} + - name: GF_PATHS_DATA + value: {{ (get .Values "grafana.ini").paths.data }} + - name: GF_PATHS_LOGS + value: {{ (get .Values "grafana.ini").paths.logs }} + - name: GF_PATHS_PLUGINS + value: {{ (get .Values "grafana.ini").paths.plugins }} + - name: GF_PATHS_PROVISIONING + value: {{ (get .Values "grafana.ini").paths.provisioning }} + {{- range $key, $value := .Values.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- range $key, $value := .Values.env }} + - name: "{{ tpl $key $ }}" + value: "{{ tpl (print $value) $ }}" + {{- end }} + {{- if or .Values.envFromSecret (or .Values.envRenderSecret .Values.envFromSecrets) .Values.envFromConfigMaps }} + envFrom: + {{- if .Values.envFromSecret }} + - secretRef: + name: {{ tpl .Values.envFromSecret . }} + {{- end }} + {{- if .Values.envRenderSecret }} + - secretRef: + name: {{ include "grafana.fullname" . }}-env + {{- end }} + {{- range .Values.envFromSecrets }} + - secretRef: + name: {{ tpl .name $ }} + optional: {{ .optional | default false }} + {{- if .prefix }} + prefix: {{ tpl .prefix $ }} + {{- end }} + {{- end }} + {{- range .Values.envFromConfigMaps }} + - configMapRef: + name: {{ tpl .name $ }} + optional: {{ .optional | default false }} + {{- if .prefix }} + prefix: {{ tpl .prefix $ }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.lifecycleHooks }} + lifecycle: + {{- tpl (toYaml .) $root | nindent 6 }} + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- with .Values.extraContainers }} + {{- tpl . $ | nindent 2 }} +{{- end }} +nodeSelector: {{ include "linux-node-selector" . | nindent 2 }} +{{- with .Values.nodeSelector }} + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.affinity }} +affinity: + {{- tpl (toYaml .) $root | nindent 2 }} +{{- end }} +{{- with .Values.topologySpreadConstraints }} +topologySpreadConstraints: + {{- toYaml . | nindent 2 }} +{{- end }} +tolerations: {{ include "linux-node-tolerations" . | nindent 2 }} +{{- with .Values.tolerations }} + {{- toYaml . | nindent 2 }} +{{- end }} +volumes: + - name: config + configMap: + name: {{ include "grafana.fullname" . }} + {{- $createConfigSecret := eq (include "grafana.shouldCreateConfigSecret" .) "true" -}} + {{- if and .Values.createConfigmap $createConfigSecret }} + - name: config-secret + secret: + secretName: {{ include "grafana.fullname" . }}-config-secret + {{- end }} + {{- range .Values.extraConfigmapMounts }} + - name: {{ tpl .name $root }} + configMap: + name: {{ tpl .configMap $root }} + {{- with .items }} + items: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.dashboards }} + {{- range (keys .Values.dashboards | sortAlpha) }} + - name: dashboards-{{ . }} + configMap: + name: {{ include "grafana.fullname" $ }}-dashboards-{{ . }} + {{- end }} + {{- end }} + {{- if .Values.dashboardsConfigMaps }} + {{- range $provider, $name := .Values.dashboardsConfigMaps }} + - name: dashboards-{{ $provider }} + configMap: + name: {{ tpl $name $root }} + {{- end }} + {{- end }} + {{- if .Values.ldap.enabled }} + - name: ldap + secret: + {{- if .Values.ldap.existingSecret }} + secretName: {{ .Values.ldap.existingSecret }} + {{- else }} + secretName: {{ include "grafana.fullname" . }} + {{- end }} + items: + - key: ldap-toml + path: ldap.toml + {{- end }} + {{- if and .Values.persistence.enabled (eq .Values.persistence.type "pvc") }} + - name: storage + persistentVolumeClaim: + claimName: {{ tpl (.Values.persistence.existingClaim | default (include "grafana.fullname" .)) . }} + {{- else if and .Values.persistence.enabled (has .Values.persistence.type $sts) }} + {{/* nothing */}} + {{- else }} + - name: storage + {{- if .Values.persistence.inMemory.enabled }} + emptyDir: + medium: Memory + {{- with .Values.persistence.inMemory.sizeLimit }} + sizeLimit: {{ . }} + {{- end }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.alerts.enabled }} + - name: sc-alerts-volume + emptyDir: + {{- with .Values.sidecar.alerts.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.dashboards.enabled }} + - name: sc-dashboard-volume + emptyDir: + {{- with .Values.sidecar.dashboards.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- if .Values.sidecar.dashboards.SCProvider }} + - name: sc-dashboard-provider + configMap: + name: {{ include "grafana.fullname" . }}-config-dashboards + {{- end }} + {{- end }} + {{- if .Values.sidecar.datasources.enabled }} + - name: sc-datasources-volume + emptyDir: + {{- with .Values.sidecar.datasources.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.plugins.enabled }} + - name: sc-plugins-volume + emptyDir: + {{- with .Values.sidecar.plugins.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.notifiers.enabled }} + - name: sc-notifiers-volume + emptyDir: + {{- with .Values.sidecar.notifiers.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- range .Values.extraSecretMounts }} + {{- if .secretName }} + - name: {{ .name }} + secret: + secretName: {{ .secretName }} + defaultMode: {{ .defaultMode }} + {{- with .items }} + items: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- else if .projected }} + - name: {{ .name }} + projected: + {{- toYaml .projected | nindent 6 }} + {{- else if .csi }} + - name: {{ .name }} + csi: + {{- toYaml .csi | nindent 6 }} + {{- end }} + {{- end }} + {{- range .Values.extraVolumes }} + - name: {{ .name }} + {{- if .existingClaim }} + persistentVolumeClaim: + claimName: {{ .existingClaim }} + {{- else if .hostPath }} + hostPath: + {{ toYaml .hostPath | nindent 6 }} + {{- else if .csi }} + csi: + {{- toYaml .csi | nindent 6 }} + {{- else if .configMap }} + configMap: + {{- toYaml .configMap | nindent 6 }} + {{- else if .emptyDir }} + emptyDir: + {{- toYaml .emptyDir | nindent 6 }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} + {{- range .Values.extraEmptyDirMounts }} + - name: {{ .name }} + emptyDir: {} + {{- end }} + {{- with .Values.extraContainerVolumes }} + {{- tpl (toYaml .) $root | nindent 2 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/clusterrole.yaml new file mode 100644 index 0000000000..3af4b62b63 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/clusterrole.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.rbac.create (or (not .Values.rbac.namespaced) .Values.rbac.extraClusterRoleRules) (not .Values.rbac.useExistingClusterRole) }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "grafana.fullname" . }}-clusterrole +{{- if or .Values.sidecar.dashboards.enabled .Values.rbac.extraClusterRoleRules .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.sidecar.alerts.enabled }} +rules: + {{- if or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.sidecar.alerts.enabled }} + - apiGroups: [""] # "" indicates the core API group + resources: ["configmaps", "secrets"] + verbs: ["get", "watch", "list"] + {{- end}} + {{- with .Values.rbac.extraClusterRoleRules }} + {{- toYaml . | nindent 2 }} + {{- end}} +{{- else }} +rules: [] +{{- end}} +{{- end}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..bda9431a2c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/clusterrolebinding.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.rbac.create (or (not .Values.rbac.namespaced) .Values.rbac.extraClusterRoleRules) }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "grafana.fullname" . }}-clusterrolebinding + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +subjects: + - kind: ServiceAccount + name: {{ include "grafana.serviceAccountName" . }} + namespace: {{ include "grafana.namespace" . }} +roleRef: + kind: ClusterRole + {{- if .Values.rbac.useExistingClusterRole }} + name: {{ .Values.rbac.useExistingClusterRole }} + {{- else }} + name: {{ include "grafana.fullname" . }}-clusterrole + {{- end }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/configSecret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/configSecret.yaml new file mode 100644 index 0000000000..55574b9bbc --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/configSecret.yaml @@ -0,0 +1,43 @@ +{{- $createConfigSecret := eq (include "grafana.shouldCreateConfigSecret" .) "true" -}} +{{- if and .Values.createConfigmap $createConfigSecret }} +{{- $files := .Files }} +{{- $root := . -}} +apiVersion: v1 +kind: Secret +metadata: + name: "{{ include "grafana.fullname" . }}-config-secret" + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: +{{- range $key, $value := .Values.alerting }} + {{- if (hasKey $value "secretFile") }} + {{- $key | nindent 2 }}: + {{- toYaml ( $files.Get $value.secretFile ) | b64enc | nindent 4}} + {{/* as of https://helm.sh/docs/chart_template_guide/accessing_files/ this will only work if you fork this chart and add files to it*/}} + {{- end }} +{{- end }} +stringData: +{{- range $key, $value := .Values.datasources }} +{{- if (hasKey $value "secret") }} +{{- $key | nindent 2 }}: | + {{- tpl (toYaml $value.secret | nindent 4) $root }} +{{- end }} +{{- end }} +{{- range $key, $value := .Values.notifiers }} +{{- if (hasKey $value "secret") }} +{{- $key | nindent 2 }}: | + {{- tpl (toYaml $value.secret | nindent 4) $root }} +{{- end }} +{{- end }} +{{- range $key, $value := .Values.alerting }} +{{ if (hasKey $value "secret") }} + {{- $key | nindent 2 }}: | + {{- tpl (toYaml $value.secret | nindent 4) $root }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/configmap-dashboard-provider.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/configmap-dashboard-provider.yaml new file mode 100644 index 0000000000..b412c4d1f0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/configmap-dashboard-provider.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.sidecar.dashboards.enabled .Values.sidecar.dashboards.SCProvider }} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "grafana.fullname" . }}-config-dashboards + namespace: {{ include "grafana.namespace" . }} +data: + {{- include "grafana.configDashboardProviderData" . | nindent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/configmap.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/configmap.yaml new file mode 100644 index 0000000000..7d7428be51 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/configmap.yaml @@ -0,0 +1,15 @@ +{{- if .Values.createConfigmap }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + {{- include "grafana.configData" . | nindent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/dashboards-json-configmap.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/dashboards-json-configmap.yaml new file mode 100644 index 0000000000..b96ce72026 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/dashboards-json-configmap.yaml @@ -0,0 +1,38 @@ +{{- if .Values.dashboards }} +{{ $files := .Files }} +{{- range $provider, $dashboards := .Values.dashboards }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "grafana.fullname" $ }}-dashboards-{{ $provider }} + namespace: {{ include "grafana.namespace" $ }} + labels: + {{- include "grafana.labels" $ | nindent 4 }} + dashboard-provider: {{ $provider }} + {{- if $.Values.sidecar.dashboards.enabled }} + {{ $.Values.sidecar.dashboards.label }}: {{ $.Values.sidecar.dashboards.labelValue | quote }} + {{- end }} +{{- if $dashboards }} +data: +{{- $dashboardFound := false }} +{{- range $key, $value := $dashboards }} +{{- if (or (hasKey $value "json") (hasKey $value "file")) }} +{{- $dashboardFound = true }} + {{- print $key | nindent 2 }}.json: + {{- if hasKey $value "json" }} + |- + {{- $value.json | nindent 6 }} + {{- end }} + {{- if hasKey $value "file" }} + {{- toYaml ( $files.Get $value.file ) | nindent 4}} + {{- end }} +{{- end }} +{{- end }} +{{- if not $dashboardFound }} + {} +{{- end }} +{{- end }} +--- +{{- end }} + +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/deployment.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/deployment.yaml new file mode 100644 index 0000000000..46c016faa3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/deployment.yaml @@ -0,0 +1,53 @@ +{{- if (and (not .Values.useStatefulSet) (or (not .Values.persistence.enabled) (eq .Values.persistence.type "pvc"))) }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and (not .Values.autoscaling.enabled) (.Values.replicas) }} + replicas: {{ .Values.replicas }} + {{- end }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + {{- with .Values.deploymentStrategy }} + strategy: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "grafana.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include "grafana.configData" . | sha256sum }} + {{- if .Values.dashboards }} + checksum/dashboards-json-config: {{ include (print $.Template.BasePath "/dashboards-json-configmap.yaml") . | sha256sum }} + {{- end }} + checksum/sc-dashboard-provider-config: {{ include "grafana.configDashboardProviderData" . | sha256sum }} + {{- if and (or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret))) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + checksum/secret: {{ include "grafana.secretsData" . | sha256sum }} + {{- end }} + {{- if .Values.envRenderSecret }} + checksum/secret-env: {{ tpl (toYaml .Values.envRenderSecret) . | sha256sum }} + {{- end }} + kubectl.kubernetes.io/default-container: {{ .Chart.Name }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- include "grafana.pod" . | nindent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/extra-manifests.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/extra-manifests.yaml new file mode 100644 index 0000000000..a9bb3b6ba8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraObjects }} +--- +{{ tpl (toYaml .) $ }} +{{ end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/headless-service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/headless-service.yaml new file mode 100644 index 0000000000..3028589d32 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/headless-service.yaml @@ -0,0 +1,22 @@ +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- if or .Values.headlessService (and .Values.persistence.enabled (not .Values.persistence.existingClaim) (has .Values.persistence.type $sts)) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "grafana.fullname" . }}-headless + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + clusterIP: None + selector: + {{- include "grafana.selectorLabels" . | nindent 4 }} + type: ClusterIP + ports: + - name: {{ .Values.gossipPortName }}-tcp + port: 9094 +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/hpa.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/hpa.yaml new file mode 100644 index 0000000000..46bbcb49a2 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/hpa.yaml @@ -0,0 +1,52 @@ +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- if .Values.autoscaling.enabled }} +apiVersion: {{ include "grafana.hpa.apiVersion" . }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + app.kubernetes.io/name: {{ include "grafana.name" . }} + helm.sh/chart: {{ include "grafana.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + {{- if has .Values.persistence.type $sts }} + kind: StatefulSet + {{- else }} + kind: Deployment + {{- end }} + name: {{ include "grafana.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetMemory }} + - type: Resource + resource: + name: memory + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.autoscaling.targetMemory }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemory }} + {{- end }} + {{- end }} + {{- if .Values.autoscaling.targetCPU }} + - type: Resource + resource: + name: cpu + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.autoscaling.targetCPU }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPU }} + {{- end }} + {{- end }} + {{- if .Values.autoscaling.behavior }} + behavior: {{ toYaml .Values.autoscaling.behavior | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-deployment.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-deployment.yaml new file mode 100644 index 0000000000..28231b803e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-deployment.yaml @@ -0,0 +1,131 @@ +{{ if .Values.imageRenderer.enabled }} +{{- $root := . -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} + {{- with .Values.imageRenderer.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.imageRenderer.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and (not .Values.imageRenderer.autoscaling.enabled) (.Values.imageRenderer.replicas) }} + replicas: {{ .Values.imageRenderer.replicas }} + {{- end }} + revisionHistoryLimit: {{ .Values.imageRenderer.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + + {{- with .Values.imageRenderer.deploymentStrategy }} + strategy: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 8 }} + {{- with .Values.imageRenderer.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.imageRenderer.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imageRenderer.schedulerName }} + schedulerName: "{{ . }}" + {{- end }} + {{- with .Values.imageRenderer.serviceAccountName }} + serviceAccountName: "{{ . }}" + {{- end }} + {{- with .Values.imageRenderer.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.imageRenderer.image.pullSecrets }} + imagePullSecrets: + {{- range . }} + - name: {{ tpl . $root }} + {{- end}} + {{- end }} + containers: + - name: {{ .Chart.Name }}-image-renderer + {{- $registry := include "system_default_registry" | default .Values.imageRenderer.image.registry -}} + {{- if .Values.imageRenderer.image.sha }} + image: "{{ $registry }}{{ .Values.imageRenderer.image.repository }}:{{ .Values.imageRenderer.image.tag }}@sha256:{{ .Values.imageRenderer.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.imageRenderer.image.repository }}:{{ .Values.imageRenderer.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.imageRenderer.image.pullPolicy }} + {{- if .Values.imageRenderer.command }} + command: + {{- range .Values.imageRenderer.command }} + - {{ . }} + {{- end }} + {{- end}} + ports: + - name: {{ .Values.imageRenderer.service.portName }} + containerPort: {{ .Values.imageRenderer.service.targetPort }} + protocol: TCP + livenessProbe: + httpGet: + path: / + port: {{ .Values.imageRenderer.service.portName }} + env: + - name: HTTP_PORT + value: {{ .Values.imageRenderer.service.targetPort | quote }} + {{- if .Values.imageRenderer.serviceMonitor.enabled }} + - name: ENABLE_METRICS + value: "true" + {{- end }} + {{- range $key, $value := .Values.imageRenderer.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 16 }} + {{- end }} + {{- range $key, $value := .Values.imageRenderer.env }} + - name: {{ $key | quote }} + value: {{ $value | quote }} + {{- end }} + {{- with .Values.imageRenderer.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - mountPath: /tmp + name: image-renderer-tmpfs + {{- with .Values.imageRenderer.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.imageRenderer.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.affinity }} + affinity: + {{- tpl (toYaml .) $root | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: image-renderer-tmpfs + emptyDir: {} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-hpa.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-hpa.yaml new file mode 100644 index 0000000000..b0f0059b79 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-hpa.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.autoscaling.enabled }} +apiVersion: {{ include "grafana.hpa.apiVersion" . }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + namespace: {{ include "grafana.namespace" . }} + labels: + app.kubernetes.io/name: {{ include "grafana.name" . }}-image-renderer + helm.sh/chart: {{ include "grafana.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "grafana.fullname" . }}-image-renderer + minReplicas: {{ .Values.imageRenderer.autoscaling.minReplicas }} + maxReplicas: {{ .Values.imageRenderer.autoscaling.maxReplicas }} + metrics: + {{- if .Values.imageRenderer.autoscaling.targetMemory }} + - type: Resource + resource: + name: memory + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.imageRenderer.autoscaling.targetMemory }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.imageRenderer.autoscaling.targetMemory }} + {{- end }} + {{- end }} + {{- if .Values.imageRenderer.autoscaling.targetCPU }} + - type: Resource + resource: + name: cpu + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.imageRenderer.autoscaling.targetCPU }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.imageRenderer.autoscaling.targetCPU }} + {{- end }} + {{- end }} + {{- if .Values.imageRenderer.autoscaling.behavior }} + behavior: {{ toYaml .Values.imageRenderer.autoscaling.behavior | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-network-policy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-network-policy.yaml new file mode 100644 index 0000000000..d1a0eb313d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-network-policy.yaml @@ -0,0 +1,79 @@ +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.networkPolicy.limitIngress }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer-ingress + namespace: {{ include "grafana.namespace" . }} + annotations: + comment: Limit image-renderer ingress traffic from grafana +spec: + podSelector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + {{- with .Values.imageRenderer.podLabels }} + {{- toYaml . | nindent 6 }} + {{- end }} + + policyTypes: + - Ingress + ingress: + - ports: + - port: {{ .Values.imageRenderer.service.targetPort }} + protocol: TCP + from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ include "grafana.namespace" . }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 14 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 14 }} + {{- end }} + {{- with .Values.imageRenderer.networkPolicy.extraIngressSelectors -}} + {{ toYaml . | nindent 8 }} + {{- end }} +{{- end }} + +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.networkPolicy.limitEgress }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer-egress + namespace: {{ include "grafana.namespace" . }} + annotations: + comment: Limit image-renderer egress traffic to grafana +spec: + podSelector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + {{- with .Values.imageRenderer.podLabels }} + {{- toYaml . | nindent 6 }} + {{- end }} + + policyTypes: + - Egress + egress: + # allow dns resolution + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + # talk only to grafana + - ports: + - port: {{ .Values.service.targetPort }} + protocol: TCP + to: + - namespaceSelector: + matchLabels: + name: {{ include "grafana.namespace" . }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 14 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 14 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-service.yaml new file mode 100644 index 0000000000..f8da127cf8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-service.yaml @@ -0,0 +1,31 @@ +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} + {{- with .Values.imageRenderer.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.imageRenderer.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + {{- with .Values.imageRenderer.service.clusterIP }} + clusterIP: {{ . }} + {{- end }} + ports: + - name: {{ .Values.imageRenderer.service.portName }} + port: {{ .Values.imageRenderer.service.port }} + protocol: TCP + targetPort: {{ .Values.imageRenderer.service.targetPort }} + {{- with .Values.imageRenderer.appProtocol }} + appProtocol: {{ . }} + {{- end }} + selector: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-servicemonitor.yaml new file mode 100644 index 0000000000..5d9f09d266 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/image-renderer-servicemonitor.yaml @@ -0,0 +1,48 @@ +{{- if .Values.imageRenderer.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + {{- if .Values.imageRenderer.serviceMonitor.namespace }} + namespace: {{ tpl .Values.imageRenderer.serviceMonitor.namespace . }} + {{- else }} + namespace: {{ include "grafana.namespace" . }} + {{- end }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} + {{- with .Values.imageRenderer.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + endpoints: + - port: {{ .Values.imageRenderer.service.portName }} + {{- with .Values.imageRenderer.serviceMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.imageRenderer.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + honorLabels: true + path: {{ .Values.imageRenderer.serviceMonitor.path }} + scheme: {{ .Values.imageRenderer.serviceMonitor.scheme }} + {{- with .Values.imageRenderer.serviceMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.imageRenderer.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + jobLabel: "{{ .Release.Name }}-image-renderer" + selector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + namespaceSelector: + matchNames: + - {{ include "grafana.namespace" . }} + {{- with .Values.imageRenderer.serviceMonitor.targetLabels }} + targetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/ingress.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/ingress.yaml new file mode 100644 index 0000000000..b2ffd81095 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/ingress.yaml @@ -0,0 +1,78 @@ +{{- if .Values.ingress.enabled -}} +{{- $ingressApiIsStable := eq (include "grafana.ingress.isStable" .) "true" -}} +{{- $ingressSupportsIngressClassName := eq (include "grafana.ingress.supportsIngressClassName" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "grafana.ingress.supportsPathType" .) "true" -}} +{{- $fullName := include "grafana.fullname" . -}} +{{- $servicePort := .Values.service.port -}} +{{- $ingressPath := .Values.ingress.path -}} +{{- $ingressPathType := .Values.ingress.pathType -}} +{{- $extraPaths := .Values.ingress.extraPaths -}} +apiVersion: {{ include "grafana.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $fullName }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.ingress.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.ingress.annotations }} + annotations: + {{- range $key, $value := . }} + {{ $key }}: {{ tpl $value $ | quote }} + {{- end }} + {{- end }} +spec: + {{- if and $ingressSupportsIngressClassName .Values.ingress.ingressClassName }} + ingressClassName: {{ .Values.ingress.ingressClassName }} + {{- end -}} + {{- with .Values.ingress.tls }} + tls: + {{- tpl (toYaml .) $ | nindent 4 }} + {{- end }} + rules: + {{- if .Values.ingress.hosts }} + {{- range .Values.ingress.hosts }} + - host: {{ tpl . $ | quote }} + http: + paths: + {{- with $extraPaths }} + {{- toYaml . | nindent 10 }} + {{- end }} + - path: {{ $ingressPath }} + {{- if $ingressSupportsPathType }} + pathType: {{ $ingressPathType }} + {{- end }} + backend: + {{- if $ingressApiIsStable }} + service: + name: {{ $fullName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end }} + {{- else }} + - http: + paths: + - backend: + {{- if $ingressApiIsStable }} + service: + name: {{ $fullName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- with $ingressPath }} + path: {{ . }} + {{- end }} + {{- if $ingressSupportsPathType }} + pathType: {{ $ingressPathType }} + {{- end }} + {{- end -}} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/networkpolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/networkpolicy.yaml new file mode 100644 index 0000000000..4cd3ed6976 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/networkpolicy.yaml @@ -0,0 +1,61 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + policyTypes: + {{- if .Values.networkPolicy.ingress }} + - Ingress + {{- end }} + {{- if .Values.networkPolicy.egress.enabled }} + - Egress + {{- end }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + + {{- if .Values.networkPolicy.egress.enabled }} + egress: + {{- if not .Values.networkPolicy.egress.blockDNSResolution }} + - ports: + - port: 53 + protocol: UDP + {{- end }} + - ports: + {{ .Values.networkPolicy.egress.ports | toJson }} + {{- with .Values.networkPolicy.egress.to }} + to: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.networkPolicy.ingress }} + ingress: + - ports: + - port: {{ .Values.service.targetPort }} + {{- if not .Values.networkPolicy.allowExternal }} + from: + - podSelector: + matchLabels: + {{ include "grafana.fullname" . }}-client: "true" + {{- with .Values.networkPolicy.explicitNamespacesSelector }} + - namespaceSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + - podSelector: + matchLabels: + {{- include "grafana.labels" . | nindent 14 }} + role: read + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/nginx-config.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/nginx-config.yaml new file mode 100644 index 0000000000..557471f6ff --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/nginx-config.yaml @@ -0,0 +1,94 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-nginx-proxy-config + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +data: + nginx.conf: |- + worker_processes auto; + error_log /dev/stdout warn; + pid /var/cache/nginx/nginx.pid; + + events { + worker_connections 1024; + } + + http { + include /etc/nginx/mime.types; + log_format main '[$time_local - $status] $remote_addr - $remote_user $request ($http_referer)'; + + proxy_connect_timeout 10; + proxy_read_timeout 180; + proxy_send_timeout 5; + proxy_buffering off; + proxy_cache_path /var/cache/nginx/cache levels=1:2 keys_zone=my_zone:100m inactive=1d max_size=10g; + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + server { + listen 8080; + access_log off; + + gzip on; + gzip_min_length 1k; + gzip_comp_level 2; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript image/jpeg image/gif image/png; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + proxy_set_header Host $host; + + location /api/dashboards { + proxy_pass http://localhost:3000; + } + + location /api/search { + proxy_pass http://localhost:3000; + + sub_filter_types application/json; + sub_filter_once off; + } + + location /api/live/ { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $http_host; + proxy_pass http://localhost:3000; + } + + location / { + proxy_cache my_zone; + proxy_cache_valid 200 302 1d; + proxy_cache_valid 301 30d; + proxy_cache_valid any 5m; + proxy_cache_bypass $http_cache_control; + add_header X-Proxy-Cache $upstream_cache_status; + add_header Cache-Control "public"; + + proxy_pass http://localhost:3000/; + + sub_filter_once off; + + {{- if eq .Values.global.cattle.clusterId "local" -}} + sub_filter '"appSubUrl":""' '"appSubUrl":"/api/v1/namespaces/{{ template "grafana.namespace" . }}/services/http:{{ template "grafana.fullname" . }}:{{ .Values.service.port }}/proxy"'; + {{- else -}} + sub_filter '"appSubUrl":""' '"appSubUrl":"/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ template "grafana.namespace" . }}/services/http:{{ template "grafana.fullname" . }}:{{ .Values.service.port }}/proxy"'; + {{- end -}} + + sub_filter ':"/avatar/' ':"avatar/'; + + if ($request_filename ~ .*\.(?:js|css|jpg|jpeg|gif|png|ico|cur|gz|svg|svgz|mp4|ogg|ogv|webm)$) { + expires 90d; + } + + rewrite ^/k8s/clusters/.*/proxy(.*) /$1 break; + + } + } + } diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/poddisruptionbudget.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/poddisruptionbudget.yaml new file mode 100644 index 0000000000..05251214ac --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/poddisruptionbudget.yaml @@ -0,0 +1,22 @@ +{{- if .Values.podDisruptionBudget }} +apiVersion: {{ include "grafana.podDisruptionBudget.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ . }} + {{- end }} + {{- with .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/podsecuritypolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/podsecuritypolicy.yaml new file mode 100644 index 0000000000..973caccd57 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/podsecuritypolicy.yaml @@ -0,0 +1,45 @@ +{{- if and (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "grafana.fullname" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- if .Values.rbac.pspAnnotations }} + annotations: {{ toYaml .Values.rbac.pspAnnotations | nindent 4 }} +{{- end }} +spec: + privileged: false + allowPrivilegeEscalation: false + requiredDropCapabilities: + # Default set from Docker, with DAC_OVERRIDE and CHOWN + - ALL + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'csi' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/pvc.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/pvc.yaml new file mode 100644 index 0000000000..c9b234305f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/pvc.yaml @@ -0,0 +1,41 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) (eq .Values.persistence.type "pvc")}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.persistence.extraPvcLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.persistence.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.persistence.finalizers }} + finalizers: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + accessModes: +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" .Values.persistence.accessModes }} +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" (first .Values.persistence.accessModes) }} + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + {{- if (lookup "v1" "PersistentVolumeClaim" (include "grafana.namespace" .) (include "grafana.fullname" .)) }} + volumeName: {{ (lookup "v1" "PersistentVolumeClaim" (include "grafana.namespace" .) (include "grafana.fullname" .)).spec.volumeName }} + {{- end }} + {{- with .Values.persistence.storageClassName }} + storageClassName: {{ . }} + {{- end }} + {{- with .Values.persistence.selectorLabels }} + selector: + matchLabels: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/role.yaml new file mode 100644 index 0000000000..469b6f4e6c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/role.yaml @@ -0,0 +1,32 @@ +{{- if and .Values.rbac.create (not .Values.rbac.useExistingRole) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if or (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) (and .Values.rbac.namespaced (or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.rbac.extraRoleRules)) }} +rules: + {{- if and (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} + - apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ include "grafana.fullname" . }}] + {{- end }} + {{- if and .Values.rbac.namespaced (or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled) }} + - apiGroups: [""] # "" indicates the core API group + resources: ["configmaps", "secrets"] + verbs: ["get", "watch", "list"] + {{- end }} + {{- with .Values.rbac.extraRoleRules }} + {{- toYaml . | nindent 2 }} + {{- end}} +{{- else }} +rules: [] +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/rolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/rolebinding.yaml new file mode 100644 index 0000000000..58f77c6b0b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/rolebinding.yaml @@ -0,0 +1,25 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + {{- if .Values.rbac.useExistingRole }} + name: {{ .Values.rbac.useExistingRole }} + {{- else }} + name: {{ include "grafana.fullname" . }} + {{- end }} +subjects: +- kind: ServiceAccount + name: {{ include "grafana.serviceAccountName" . }} + namespace: {{ include "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/secret-env.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/secret-env.yaml new file mode 100644 index 0000000000..eb14aac707 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/secret-env.yaml @@ -0,0 +1,14 @@ +{{- if .Values.envRenderSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "grafana.fullname" . }}-env + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +type: Opaque +data: +{{- range $key, $val := .Values.envRenderSecret }} + {{ $key }}: {{ tpl ($val | toString) $ | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/secret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/secret.yaml new file mode 100644 index 0000000000..fd2ca50f4b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/secret.yaml @@ -0,0 +1,16 @@ +{{- if or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret)) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + {{- include "grafana.secretsData" . | nindent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/service.yaml new file mode 100644 index 0000000000..e9396a15c6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/service.yaml @@ -0,0 +1,61 @@ +{{- if .Values.service.enabled }} +{{- $root := . }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.service.annotations }} + annotations: + {{- tpl (toYaml . | nindent 4) $root }} + {{- end }} +spec: + {{- if (or (eq .Values.service.type "ClusterIP") (empty .Values.service.type)) }} + type: ClusterIP + {{- with .Values.service.clusterIP }} + clusterIP: {{ . }} + {{- end }} + {{- else if eq .Values.service.type "LoadBalancer" }} + type: LoadBalancer + {{- with .Values.service.loadBalancerIP }} + loadBalancerIP: {{ . }} + {{- end }} + {{- with .Values.service.loadBalancerClass }} + loadBalancerClass: {{ . }} + {{- end }} + {{- with .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- else }} + type: {{ .Values.service.type }} + {{- end }} + {{- with .Values.service.externalIPs }} + externalIPs: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.service.externalTrafficPolicy }} + externalTrafficPolicy: {{ . }} + {{- end }} + ports: + - name: {{ .Values.service.portName }} + port: {{ .Values.service.port }} + protocol: TCP + targetPort: {{ .Values.service.targetPort }} + {{- with .Values.service.appProtocol }} + appProtocol: {{ . }} + {{- end }} + {{- if (and (eq .Values.service.type "NodePort") (not (empty .Values.service.nodePort))) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + {{- with .Values.extraExposePorts }} + {{- tpl (toYaml . | nindent 4) $root }} + {{- end }} + selector: + {{- include "grafana.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/serviceaccount.yaml new file mode 100644 index 0000000000..ffca0717ae --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.serviceAccount.autoMount | default .Values.serviceAccount.automountServiceAccountToken }} +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} + name: {{ include "grafana.serviceAccountName" . }} + namespace: {{ include "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/servicemonitor.yaml new file mode 100644 index 0000000000..b321b1269c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/servicemonitor.yaml @@ -0,0 +1,68 @@ +{{- if .Values.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "grafana.fullname" . }} + {{- if .Values.serviceMonitor.namespace }} + namespace: {{ tpl .Values.serviceMonitor.namespace . }} + {{- else }} + namespace: {{ include "grafana.namespace" . }} + {{- end }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.serviceMonitor.labels }} + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} +spec: + endpoints: + - port: {{ .Values.service.portName }} + {{- with .Values.serviceMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + honorLabels: true + path: {{ .Values.serviceMonitor.path }} + scheme: {{ .Values.serviceMonitor.scheme }} + {{- with .Values.serviceMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 6 }} + {{- end }} + metricRelabelings: + {{- if .Values.serviceMonitor.metricRelabelings }} + {{- toYaml .Values.serviceMonitor.metricRelabelings | nindent 6 }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.serviceMonitor.relabelings }} + {{- with .Values.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- end }} + {{- with .Values.serviceMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + jobLabel: "{{ .Release.Name }}" + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + namespaceSelector: + matchNames: + - {{ include "grafana.namespace" . }} + {{- with .Values.serviceMonitor.targetLabels }} + targetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/statefulset.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/statefulset.yaml new file mode 100644 index 0000000000..49278083e8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/statefulset.yaml @@ -0,0 +1,58 @@ +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- if (or (.Values.useStatefulSet) (and .Values.persistence.enabled (not .Values.persistence.existingClaim) (has .Values.persistence.type $sts)))}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.replicas }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + serviceName: {{ include "grafana.fullname" . }}-headless + template: + metadata: + labels: + {{- include "grafana.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + checksum/dashboards-json-config: {{ include (print $.Template.BasePath "/dashboards-json-configmap.yaml") . | sha256sum }} + checksum/sc-dashboard-provider-config: {{ include (print $.Template.BasePath "/configmap-dashboard-provider.yaml") . | sha256sum }} + {{- if and (or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret))) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- end }} + kubectl.kubernetes.io/default-container: {{ .Chart.Name }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- include "grafana.pod" . | nindent 6 }} + {{- if .Values.persistence.enabled}} + volumeClaimTemplates: + - metadata: + name: storage + spec: +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" .Values.persistence.accessModes }} +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" (first .Values.persistence.accessModes) }} + accessModes: {{ .Values.persistence.accessModes }} + storageClassName: {{ .Values.persistence.storageClassName }} + resources: + requests: + storage: {{ required "Must provide size for persistent volumes used by Grafana" .Values.persistence.size }} + {{- with .Values.persistence.selectorLabels }} + selector: + matchLabels: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-configmap.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-configmap.yaml new file mode 100644 index 0000000000..01c96c9243 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-configmap.yaml @@ -0,0 +1,20 @@ +{{- if .Values.testFramework.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "grafana.fullname" . }}-test + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +data: + run.sh: |- + @test "Test Health" { + url="http://{{ include "grafana.fullname" . }}/api/health" + + code=$(wget --server-response --spider --timeout 90 --tries 10 ${url} 2>&1 | awk '/^ HTTP/{print $2}') + [ "$code" == "200" ] + } +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml new file mode 100644 index 0000000000..70a0a884c9 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml @@ -0,0 +1,32 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.testFramework.enabled (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "grafana.fullname" . }}-test + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +spec: + allowPrivilegeEscalation: true + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + fsGroup: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + runAsUser: + rule: RunAsAny + volumes: + - configMap + - downwardAPI + - emptyDir + - projected + - csi + - secret +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-role.yaml new file mode 100644 index 0000000000..976418b137 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-role.yaml @@ -0,0 +1,17 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.testFramework.enabled (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "grafana.fullname" . }}-test + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +rules: + - apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ include "grafana.fullname" . }}-test] +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-rolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-rolebinding.yaml new file mode 100644 index 0000000000..509566eccd --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-rolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.testFramework.enabled (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "grafana.fullname" . }}-test + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "grafana.fullname" . }}-test +subjects: + - kind: ServiceAccount + name: {{ include "grafana.serviceAccountNameTest" . }} + namespace: {{ include "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-serviceaccount.yaml new file mode 100644 index 0000000000..38fba3596a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.testFramework.enabled .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + name: {{ include "grafana.serviceAccountNameTest" . }} + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test.yaml new file mode 100644 index 0000000000..83aaa185c2 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/templates/tests/test.yaml @@ -0,0 +1,53 @@ +{{- if .Values.testFramework.enabled }} +{{- $root := . }} +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "grafana.fullname" . }}-test + labels: + {{- include "grafana.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + namespace: {{ include "grafana.namespace" . }} +spec: + serviceAccountName: {{ include "grafana.serviceAccountNameTest" . }} + {{- with .Values.testFramework.securityContext }} + securityContext: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if or .Values.image.pullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "grafana.imagePullSecrets" (dict "root" $root "imagePullSecrets" .Values.image.pullSecrets) | nindent 4 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- tpl (toYaml .) $root | nindent 4 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 4 }} + {{- end }} + containers: + - name: {{ .Release.Name }}-test + image: "{{ template "system_default_registry" . | default .Values.testFramework.image.registry }}/{{ .Values.testFramework.image.repository }}:{{ .Values.testFramework.image.tag }}" + imagePullPolicy: "{{ .Values.testFramework.imagePullPolicy}}" + command: ["/opt/bats/bin/bats", "-t", "/tests/run.sh"] + volumeMounts: + - mountPath: /tests + name: tests + readOnly: true + {{- with .Values.testFramework.resources }} + resources: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: tests + configMap: + name: {{ include "grafana.fullname" . }}-test + restartPolicy: Never +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/values.yaml new file mode 100644 index 0000000000..2bb62aec39 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/grafana/values.yaml @@ -0,0 +1,1315 @@ +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # Can be tempalted. + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + +rbac: + create: true + ## Use an existing ClusterRole/Role (depending on rbac.namespaced false/true) + # useExistingRole: name-of-some-role + # useExistingClusterRole: name-of-some-clusterRole + pspEnabled: false + pspUseAppArmor: false + namespaced: false + extraRoleRules: [] + # - apiGroups: [] + # resources: [] + # verbs: [] + extraClusterRoleRules: [] + # - apiGroups: [] + # resources: [] + # verbs: [] +serviceAccount: + create: true + name: + nameTest: + ## ServiceAccount labels. + labels: {} + ## Service account annotations. Can be templated. + # annotations: + # eks.amazonaws.com/role-arn: arn:aws:iam::123456789000:role/iam-role-name-here + + ## autoMount is deprecated in favor of automountServiceAccountToken + # autoMount: false + automountServiceAccountToken: true + +replicas: 1 + +## Create a headless service for the deployment +headlessService: false + +## Should the service account be auto mounted on the pod +automountServiceAccountToken: true + +## Create HorizontalPodAutoscaler object for deployment type +# +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPU: "60" + targetMemory: "" + behavior: {} + +## See `kubectl explain poddisruptionbudget.spec` for more +## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ +podDisruptionBudget: {} +# apiVersion: "" +# minAvailable: 1 +# maxUnavailable: 1 + +## See `kubectl explain deployment.spec.strategy` for more +## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy +deploymentStrategy: + type: RollingUpdate + +readinessProbe: + httpGet: + path: /api/health + port: 3000 + +livenessProbe: + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 60 + timeoutSeconds: 30 + failureThreshold: 10 + +## Use an alternate scheduler, e.g. "stork". +## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ +## +# schedulerName: "default-scheduler" + +image: + repository: rancher/mirrored-grafana-grafana + # Overrides the Grafana image tag whose default is the chart appVersion + tag: 10.3.3 + sha: "" + pullPolicy: IfNotPresent + + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Can be templated. + ## + pullSecrets: [] + # - myRegistrKeySecretName + +testFramework: + enabled: false + imagePullPolicy: IfNotPresent + securityContext: + runAsNonRoot: true + runAsUser: 1000 + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +# dns configuration for pod +dnsPolicy: ~ +dnsConfig: {} + # nameservers: + # - 8.8.8.8 + # options: + # - name: ndots + # value: "2" + # - name: edns0 + +securityContext: + runAsNonRoot: true + runAsUser: 472 + runAsGroup: 472 + fsGroup: 472 + +containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + +# Enable creating the grafana configmap +createConfigmap: true + +# Extra configmaps to mount in grafana pods +# Values are templated. +extraConfigmapMounts: [] + # - name: certs-configmap + # mountPath: /etc/grafana/ssl/ + # subPath: certificates.crt # (optional) + # configMap: certs-configmap + # readOnly: true + + +extraEmptyDirMounts: [] + # - name: provisioning-notifiers + # mountPath: /etc/grafana/provisioning/notifiers + + +# Apply extra labels to common labels. +extraLabels: {} + +## Assign a PriorityClassName to pods if set +# priorityClassName: + +downloadDashboardsImage: + repository: rancher/mirrored-curlimages-curl + tag: 7.85.0 + sha: "" + pullPolicy: IfNotPresent + +downloadDashboards: + env: {} + envFromSecret: "" + resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + +## Pod Annotations +# podAnnotations: {} + +## Pod Labels +# podLabels: {} + +podPortName: grafana +gossipPortName: gossip +## Deployment annotations +# annotations: {} + +## Expose the grafana service to be accessed from outside the cluster (LoadBalancer service). +## or access it from within the cluster (ClusterIP service). Set the service type and the port to serve it. +## ref: http://kubernetes.io/docs/user-guide/services/ +## +service: + enabled: true + type: ClusterIP + loadBalancerIP: "" + loadBalancerClass: "" + loadBalancerSourceRanges: [] + port: 80 + targetPort: 3000 + # targetPort: 4181 To be used with a proxy extraContainer + ## Service annotations. Can be templated. + annotations: {} + labels: {} + portName: service + # Adds the appProtocol field to the service. This allows to work with istio protocol selection. Ex: "http" or "tcp" + appProtocol: "" + +serviceMonitor: + ## If true, a ServiceMonitor CRD is created for a prometheus operator + ## https://github.com/coreos/prometheus-operator + ## + enabled: false + path: /metrics + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + labels: {} + interval: 30s + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + relabelings: [] + metricRelabelings: [] + targetLabels: [] + +extraExposePorts: [] + # - name: keycloak + # port: 8080 + # targetPort: 8080 + +# overrides pod.spec.hostAliases in the grafana deployment's pods +hostAliases: [] + # - ip: "1.2.3.4" + # hostnames: + # - "my.host.com" + +ingress: + enabled: false + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + # Values can be templated + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + labels: {} + path: / + + # pathType is only for k8s >= 1.1= + pathType: Prefix + + hosts: + - chart-example.local + ## Extra paths to prepend to every host configuration. This is useful when working with annotation based services. + extraPaths: [] + # - path: /* + # backend: + # serviceName: ssl-redirect + # servicePort: use-annotation + ## Or for k8s > 1.19 + # - path: /* + # pathType: Prefix + # backend: + # service: + # name: ssl-redirect + # port: + # name: use-annotation + + + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} +# limits: +# cpu: 100m +# memory: 128Mi +# requests: +# cpu: 100m +# memory: 128Mi + +## Node labels for pod assignment +## ref: https://kubernetes.io/docs/user-guide/node-selection/ +# +nodeSelector: {} + +## Tolerations for pod assignment +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## Affinity for pod assignment (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## Topology Spread Constraints +## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ +## +topologySpreadConstraints: [] + +## Additional init containers (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ +## +extraInitContainers: [] + +## Enable an Specify container in extraContainers. This is meant to allow adding an authentication proxy to a grafana pod +extraContainers: "" +# extraContainers: | +# - name: proxy +# image: quay.io/gambol99/keycloak-proxy:latest +# args: +# - -provider=github +# - -client-id= +# - -client-secret= +# - -github-org= +# - -email-domain=* +# - -cookie-secret= +# - -http-address=http://0.0.0.0:4181 +# - -upstream-url=http://127.0.0.1:3000 +# ports: +# - name: proxy-web +# containerPort: 4181 + +## Volumes that can be used in init containers that will not be mounted to deployment pods +extraContainerVolumes: [] +# - name: volume-from-secret +# secret: +# secretName: secret-to-mount +# - name: empty-dir-volume +# emptyDir: {} + +## Enable persistence using Persistent Volume Claims +## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ +## +persistence: + type: pvc + enabled: false + # storageClassName: default + accessModes: + - ReadWriteOnce + size: 10Gi + # annotations: {} + finalizers: + - kubernetes.io/pvc-protection + # selectorLabels: {} + ## Sub-directory of the PV to mount. Can be templated. + # subPath: "" + ## Name of an existing PVC. Can be templated. + # existingClaim: + ## Extra labels to apply to a PVC. + extraPvcLabels: {} + + ## If persistence is not enabled, this allows to mount the + ## local storage in-memory to improve performance + ## + inMemory: + enabled: false + ## The maximum usage on memory medium EmptyDir would be + ## the minimum value between the SizeLimit specified + ## here and the sum of memory limits of all containers in a pod + ## + # sizeLimit: 300Mi + +initChownData: + ## If false, data ownership will not be reset at startup + ## This allows the grafana-server to be run with an arbitrary user + ## + enabled: true + + ## initChownData container image + ## + image: + repository: rancher/mirrored-library-busybox + tag: "1.31.1" + sha: "" + pullPolicy: IfNotPresent + + ## initChownData resource requests and limits + ## Ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + securityContext: + runAsNonRoot: false + runAsUser: 0 + seccompProfile: + type: RuntimeDefault + capabilities: + add: + - CHOWN + +# Administrator credentials when not using an existing secret (see below) +adminUser: admin +# adminPassword: strongpassword + +# Use an existing secret for the admin user. +admin: + ## Name of the secret. Can be templated. + existingSecret: "" + userKey: admin-user + passwordKey: admin-password + +## Define command to be executed at startup by grafana container +## Needed if using `vault-env` to manage secrets (ref: https://banzaicloud.com/blog/inject-secrets-into-pods-vault/) +## Default is "run.sh" as defined in grafana's Dockerfile +# command: +# - "sh" +# - "/run.sh" + +## Optionally define args if command is used +## Needed if using `hashicorp/envconsul` to manage secrets +## By default no arguments are set +# args: +# - "-secret" +# - "secret/grafana" +# - "./grafana" + +## Extra environment variables that will be pass onto deployment pods +## +## to provide grafana with access to CloudWatch on AWS EKS: +## 1. create an iam role of type "Web identity" with provider oidc.eks.* (note the provider for later) +## 2. edit the "Trust relationships" of the role, add a line inside the StringEquals clause using the +## same oidc eks provider as noted before (same as the existing line) +## also, replace NAMESPACE and prometheus-operator-grafana with the service account namespace and name +## +## "oidc.eks.us-east-1.amazonaws.com/id/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX:sub": "system:serviceaccount:NAMESPACE:prometheus-operator-grafana", +## +## 3. attach a policy to the role, you can use a built in policy called CloudWatchReadOnlyAccess +## 4. use the following env: (replace 123456789000 and iam-role-name-here with your aws account number and role name) +## +## env: +## AWS_ROLE_ARN: arn:aws:iam::123456789000:role/iam-role-name-here +## AWS_WEB_IDENTITY_TOKEN_FILE: /var/run/secrets/eks.amazonaws.com/serviceaccount/token +## AWS_REGION: us-east-1 +## +## 5. uncomment the EKS section in extraSecretMounts: below +## 6. uncomment the annotation section in the serviceAccount: above +## make sure to replace arn:aws:iam::123456789000:role/iam-role-name-here with your role arn + +env: {} + +## "valueFrom" environment variable references that will be added to deployment pods. Name is templated. +## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#envvarsource-v1-core +## Renders in container spec as: +## env: +## ... +## - name: +## valueFrom: +## +envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + +## The name of a secret in the same kubernetes namespace which contain values to be added to the environment +## This can be useful for auth tokens, etc. Value is templated. +envFromSecret: "" + +## Sensible environment variables that will be rendered as new secret object +## This can be useful for auth tokens, etc. +## If the secret values contains "{{", they'll need to be properly escaped so that they are not interpreted by Helm +## ref: https://helm.sh/docs/howto/charts_tips_and_tricks/#using-the-tpl-function +envRenderSecret: {} + +## The names of secrets in the same kubernetes namespace which contain values to be added to the environment +## Each entry should contain a name key, and can optionally specify whether the secret must be defined with an optional key. +## Name is templated. +envFromSecrets: [] +## - name: secret-name +## prefix: prefix +## optional: true + +## The names of conifgmaps in the same kubernetes namespace which contain values to be added to the environment +## Each entry should contain a name key, and can optionally specify whether the configmap must be defined with an optional key. +## Name is templated. +## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#configmapenvsource-v1-core +envFromConfigMaps: [] +## - name: configmap-name +## prefix: prefix +## optional: true + +# Inject Kubernetes services as environment variables. +# See https://kubernetes.io/docs/concepts/services-networking/connect-applications-service/#environment-variables +enableServiceLinks: true + +## Additional grafana server secret mounts +# Defines additional mounts with secrets. Secrets must be manually created in the namespace. +extraSecretMounts: [] + # - name: secret-files + # mountPath: /etc/secrets + # secretName: grafana-secret-files + # readOnly: true + # subPath: "" + # + # for AWS EKS (cloudwatch) use the following (see also instruction in env: above) + # - name: aws-iam-token + # mountPath: /var/run/secrets/eks.amazonaws.com/serviceaccount + # readOnly: true + # projected: + # defaultMode: 420 + # sources: + # - serviceAccountToken: + # audience: sts.amazonaws.com + # expirationSeconds: 86400 + # path: token + # + # for CSI e.g. Azure Key Vault use the following + # - name: secrets-store-inline + # mountPath: /run/secrets + # readOnly: true + # csi: + # driver: secrets-store.csi.k8s.io + # readOnly: true + # volumeAttributes: + # secretProviderClass: "akv-grafana-spc" + # nodePublishSecretRef: # Only required when using service principal mode + # name: grafana-akv-creds # Only required when using service principal mode + +## Additional grafana server volume mounts +# Defines additional volume mounts. +extraVolumeMounts: [] + # - name: extra-volume-0 + # mountPath: /mnt/volume0 + # readOnly: true + # - name: extra-volume-1 + # mountPath: /mnt/volume1 + # readOnly: true + # - name: grafana-secrets + # mountPath: /mnt/volume2 + +## Additional Grafana server volumes +extraVolumes: [] + # - name: extra-volume-0 + # existingClaim: volume-claim + # - name: extra-volume-1 + # hostPath: + # path: /usr/shared/ + # type: "" + # - name: grafana-secrets + # csi: + # driver: secrets-store.csi.k8s.io + # readOnly: true + # volumeAttributes: + # secretProviderClass: "grafana-env-spc" + +## Container Lifecycle Hooks. Execute a specific bash command or make an HTTP request +lifecycleHooks: {} + # postStart: + # exec: + # command: [] + +## Pass the plugins you want installed as a list. +## +plugins: [] + # - digrich-bubblechart-panel + # - grafana-clock-panel + ## You can also use other plugin download URL, as long as they are valid zip files, + ## and specify the name of the plugin after the semicolon. Like this: + # - https://grafana.com/api/plugins/marcusolsson-json-datasource/versions/1.3.2/download;marcusolsson-json-datasource + +## Configure grafana datasources +## ref: http://docs.grafana.org/administration/provisioning/#datasources +## +datasources: {} +# datasources.yaml: +# apiVersion: 1 +# datasources: +# - name: Prometheus +# type: prometheus +# url: http://prometheus-prometheus-server +# access: proxy +# isDefault: true +# - name: CloudWatch +# type: cloudwatch +# access: proxy +# uid: cloudwatch +# editable: false +# jsonData: +# authType: default +# defaultRegion: us-east-1 +# deleteDatasources: [] +# - name: Prometheus + +## Configure grafana alerting (can be templated) +## ref: http://docs.grafana.org/administration/provisioning/#alerting +## +alerting: {} + # rules.yaml: + # apiVersion: 1 + # groups: + # - orgId: 1 + # name: '{{ .Chart.Name }}_my_rule_group' + # folder: my_first_folder + # interval: 60s + # rules: + # - uid: my_id_1 + # title: my_first_rule + # condition: A + # data: + # - refId: A + # datasourceUid: '-100' + # model: + # conditions: + # - evaluator: + # params: + # - 3 + # type: gt + # operator: + # type: and + # query: + # params: + # - A + # reducer: + # type: last + # type: query + # datasource: + # type: __expr__ + # uid: '-100' + # expression: 1==0 + # intervalMs: 1000 + # maxDataPoints: 43200 + # refId: A + # type: math + # dashboardUid: my_dashboard + # panelId: 123 + # noDataState: Alerting + # for: 60s + # annotations: + # some_key: some_value + # labels: + # team: sre_team_1 + # contactpoints.yaml: + # secret: + # apiVersion: 1 + # contactPoints: + # - orgId: 1 + # name: cp_1 + # receivers: + # - uid: first_uid + # type: pagerduty + # settings: + # integrationKey: XXX + # severity: critical + # class: ping failure + # component: Grafana + # group: app-stack + # summary: | + # {{ `{{ include "default.message" . }}` }} + +## Configure notifiers +## ref: http://docs.grafana.org/administration/provisioning/#alert-notification-channels +## +notifiers: {} +# notifiers.yaml: +# notifiers: +# - name: email-notifier +# type: email +# uid: email1 +# # either: +# org_id: 1 +# # or +# org_name: Main Org. +# is_default: true +# settings: +# addresses: an_email_address@example.com +# delete_notifiers: + +## Configure grafana dashboard providers +## ref: http://docs.grafana.org/administration/provisioning/#dashboards +## +## `path` must be /var/lib/grafana/dashboards/ +## +dashboardProviders: {} +# dashboardproviders.yaml: +# apiVersion: 1 +# providers: +# - name: 'default' +# orgId: 1 +# folder: '' +# type: file +# disableDeletion: false +# editable: true +# options: +# path: /var/lib/grafana/dashboards/default + +## Configure grafana dashboard to import +## NOTE: To use dashboards you must also enable/configure dashboardProviders +## ref: https://grafana.com/dashboards +## +## dashboards per provider, use provider name as key. +## +dashboards: {} + # default: + # some-dashboard: + # json: | + # $RAW_JSON + # custom-dashboard: + # file: dashboards/custom-dashboard.json + # prometheus-stats: + # gnetId: 2 + # revision: 2 + # datasource: Prometheus + # local-dashboard: + # url: https://example.com/repository/test.json + # token: '' + # local-dashboard-base64: + # url: https://example.com/repository/test-b64.json + # token: '' + # b64content: true + # local-dashboard-gitlab: + # url: https://example.com/repository/test-gitlab.json + # gitlabToken: '' + # local-dashboard-bitbucket: + # url: https://example.com/repository/test-bitbucket.json + # bearerToken: '' + # local-dashboard-azure: + # url: https://example.com/repository/test-azure.json + # basic: '' + # acceptHeader: '*/*' + +## Reference to external ConfigMap per provider. Use provider name as key and ConfigMap name as value. +## A provider dashboards must be defined either by external ConfigMaps or in values.yaml, not in both. +## ConfigMap data example: +## +## data: +## example-dashboard.json: | +## RAW_JSON +## +dashboardsConfigMaps: {} +# default: "" + +## Grafana's primary configuration +## NOTE: values in map will be converted to ini format +## ref: http://docs.grafana.org/installation/configuration/ +## +grafana.ini: + paths: + data: /var/lib/grafana/ + logs: /var/log/grafana + plugins: /var/lib/grafana/plugins + provisioning: /etc/grafana/provisioning + analytics: + check_for_updates: true + log: + mode: console + grafana_net: + url: https://grafana.net + server: + domain: "{{ if (and .Values.ingress.enabled .Values.ingress.hosts) }}{{ .Values.ingress.hosts | first }}{{ else }}''{{ end }}" +## grafana Authentication can be enabled with the following values on grafana.ini + # server: + # The full public facing url you use in browser, used for redirects and emails + # root_url: + # https://grafana.com/docs/grafana/latest/auth/github/#enable-github-in-grafana + # auth.github: + # enabled: false + # allow_sign_up: false + # scopes: user:email,read:org + # auth_url: https://github.com/login/oauth/authorize + # token_url: https://github.com/login/oauth/access_token + # api_url: https://api.github.com/user + # team_ids: + # allowed_organizations: + # client_id: + # client_secret: +## LDAP Authentication can be enabled with the following values on grafana.ini +## NOTE: Grafana will fail to start if the value for ldap.toml is invalid + # auth.ldap: + # enabled: true + # allow_sign_up: true + # config_file: /etc/grafana/ldap.toml + +## Grafana's LDAP configuration +## Templated by the template in _helpers.tpl +## NOTE: To enable the grafana.ini must be configured with auth.ldap.enabled +## ref: http://docs.grafana.org/installation/configuration/#auth-ldap +## ref: http://docs.grafana.org/installation/ldap/#configuration +ldap: + enabled: false + # `existingSecret` is a reference to an existing secret containing the ldap configuration + # for Grafana in a key `ldap-toml`. + existingSecret: "" + # `config` is the content of `ldap.toml` that will be stored in the created secret + config: "" + # config: |- + # verbose_logging = true + + # [[servers]] + # host = "my-ldap-server" + # port = 636 + # use_ssl = true + # start_tls = false + # ssl_skip_verify = false + # bind_dn = "uid=%s,ou=users,dc=myorg,dc=com" + +## Grafana's SMTP configuration +## NOTE: To enable, grafana.ini must be configured with smtp.enabled +## ref: http://docs.grafana.org/installation/configuration/#smtp +smtp: + # `existingSecret` is a reference to an existing secret containing the smtp configuration + # for Grafana. + existingSecret: "" + userKey: "user" + passwordKey: "password" + +## Sidecars that collect the configmaps with specified label and stores the included files them into the respective folders +## Requires at least Grafana 5 to work and can't be used together with parameters dashboardProviders, datasources and dashboards +sidecar: + image: + repository: rancher/mirrored-kiwigrid-k8s-sidecar + tag: 1.26.1 + sha: "" + imagePullPolicy: IfNotPresent + resources: {} +# limits: +# cpu: 100m +# memory: 100Mi +# requests: +# cpu: 50m +# memory: 50Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + # skipTlsVerify Set to true to skip tls verification for kube api calls + # skipTlsVerify: true + enableUniqueFilenames: false + readinessProbe: {} + livenessProbe: {} + # Log level default for all sidecars. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. Defaults to INFO + # logLevel: INFO + alerts: + enabled: false + # Additional environment variables for the alerts sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with alert are marked with + label: grafana_alert + # value of label that the configmaps with alert are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for alert config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload alerts + reloadURL: "http://localhost:3000/api/admin/provisioning/alerting/reload" + # Absolute path to shell script to execute after a alert got reloaded + script: null + skipReload: false + # This is needed if skipReload is true, to load any alerts defined at startup time. + # Deploy the alert sidecar as an initContainer. + initAlerts: false + # Additional alert sidecar volume mounts + extraMounts: [] + # Sets the size limit of the alert sidecar emptyDir volume + sizeLimit: {} + dashboards: + enabled: false + # Additional environment variables for the dashboards sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + SCProvider: true + # label that the configmaps with dashboards are marked with + label: grafana_dashboard + # value of label that the configmaps with dashboards are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # folder in the pod that should hold the collected dashboards (unless `defaultFolderName` is set) + folder: /tmp/dashboards + # The default folder name, it will create a subfolder under the `folder` and put dashboards in there instead + defaultFolderName: null + # Namespaces list. If specified, the sidecar will search for config-maps/secrets inside these namespaces. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces. + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # If specified, the sidecar will look for annotation with this name to create folder and put graph here. + # You can use this parameter together with `provider.foldersFromFilesStructure`to annotate configmaps and create folder structure. + folderAnnotation: null + # Endpoint to send request to reload alerts + reloadURL: "http://localhost:3000/api/admin/provisioning/dashboards/reload" + # Absolute path to shell script to execute after a configmap got reloaded + script: null + skipReload: false + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # provider configuration that lets grafana manage the dashboards + provider: + # name of the provider, should be unique + name: sidecarProvider + # orgid as configured in grafana + orgid: 1 + # folder in which the dashboards should be imported in grafana + folder: '' + # type of the provider + type: file + # disableDelete to activate a import-only behaviour + disableDelete: false + # allow updating provisioned dashboards from the UI + allowUiUpdates: false + # allow Grafana to replicate dashboard structure from filesystem + foldersFromFilesStructure: false + # Additional dashboard sidecar volume mounts + extraMounts: [] + # Sets the size limit of the dashboard sidecar emptyDir volume + sizeLimit: {} + datasources: + enabled: false + # Additional environment variables for the datasourcessidecar + env: {} + envValueFrom: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with datasources are marked with + label: grafana_datasource + # value of label that the configmaps with datasources are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for datasource config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload datasources + reloadURL: "http://localhost:3000/api/admin/provisioning/datasources/reload" + # Absolute path to shell script to execute after a datasource got reloaded + script: null + skipReload: true + # This is needed if skipReload is true, to load any datasources defined at startup time. + # Deploy the datasources sidecar as an initContainer. + initDatasources: true + # Sets the size limit of the datasource sidecar emptyDir volume + sizeLimit: {} + plugins: + enabled: false + # Additional environment variables for the plugins sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with plugins are marked with + label: grafana_plugin + # value of label that the configmaps with plugins are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for plugin config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload plugins + reloadURL: "http://localhost:3000/api/admin/provisioning/plugins/reload" + # Absolute path to shell script to execute after a plugin got reloaded + script: null + skipReload: false + # Deploy the datasource sidecar as an initContainer in addition to a container. + # This is needed if skipReload is true, to load any plugins defined at startup time. + initPlugins: false + # Sets the size limit of the plugin sidecar emptyDir volume + sizeLimit: {} + notifiers: + enabled: false + # Additional environment variables for the notifierssidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with notifiers are marked with + label: grafana_notifier + # value of label that the configmaps with notifiers are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for notifier config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload notifiers + reloadURL: "http://localhost:3000/api/admin/provisioning/notifications/reload" + # Absolute path to shell script to execute after a notifier got reloaded + script: null + skipReload: false + # Deploy the notifier sidecar as an initContainer in addition to a container. + # This is needed if skipReload is true, to load any notifiers defined at startup time. + initNotifiers: false + # Sets the size limit of the notifier sidecar emptyDir volume + sizeLimit: {} + +## Override the deployment namespace +## +namespaceOverride: "" + +## Number of old ReplicaSets to retain +## +revisionHistoryLimit: 10 + +## Add a seperate remote image renderer deployment/service +imageRenderer: + deploymentStrategy: {} + # Enable the image-renderer deployment & service + enabled: false + replicas: 1 + autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPU: "60" + targetMemory: "" + behavior: {} + image: + # image-renderer Image repository + repository: rancher/mirrored-grafana-grafana-image-renderer + # image-renderer Image tag + tag: 3.10.1 + # image-renderer Image sha (optional) + sha: "" + # image-renderer ImagePullPolicy + pullPolicy: Always + # extra environment variables + env: + HTTP_HOST: "0.0.0.0" + # RENDERING_ARGS: --no-sandbox,--disable-gpu,--window-size=1280x758 + # RENDERING_MODE: clustered + # IGNORE_HTTPS_ERRORS: true + + ## "valueFrom" environment variable references that will be added to deployment pods. Name is templated. + ## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#envvarsource-v1-core + ## Renders in container spec as: + ## env: + ## ... + ## - name: + ## valueFrom: + ## + envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + + # image-renderer deployment serviceAccount + serviceAccountName: "" + # image-renderer deployment securityContext + securityContext: {} + # image-renderer deployment container securityContext + containerSecurityContext: + seccompProfile: + type: RuntimeDefault + capabilities: + drop: ['ALL'] + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + ## image-renderer pod annotation + podAnnotations: {} + # image-renderer deployment Host Aliases + hostAliases: [] + # image-renderer deployment priority class + priorityClassName: '' + service: + # Enable the image-renderer service + enabled: true + # image-renderer service port name + portName: 'http' + # image-renderer service port used by both service and deployment + port: 8081 + targetPort: 8081 + # Adds the appProtocol field to the image-renderer service. This allows to work with istio protocol selection. Ex: "http" or "tcp" + appProtocol: "" + serviceMonitor: + ## If true, a ServiceMonitor CRD is created for a prometheus operator + ## https://github.com/coreos/prometheus-operator + ## + enabled: false + path: /metrics + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + labels: {} + interval: 1m + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + relabelings: [] + # See: https://doc.crds.dev/github.com/prometheus-operator/kube-prometheus/monitoring.coreos.com/ServiceMonitor/v1@v0.11.0#spec-targetLabels + targetLabels: [] + # - targetLabel1 + # - targetLabel2 + # If https is enabled in Grafana, this needs to be set as 'https' to correctly configure the callback used in Grafana + grafanaProtocol: http + # In case a sub_path is used this needs to be added to the image renderer callback + grafanaSubPath: "" + # name of the image-renderer port on the pod + podPortName: http + # number of image-renderer replica sets to keep + revisionHistoryLimit: 10 + networkPolicy: + # Enable a NetworkPolicy to limit inbound traffic to only the created grafana pods + limitIngress: true + # Enable a NetworkPolicy to limit outbound traffic to only the created grafana pods + limitEgress: false + # Allow additional services to access image-renderer (eg. Prometheus operator when ServiceMonitor is enabled) + extraIngressSelectors: [] + resources: {} +# limits: +# cpu: 100m +# memory: 100Mi +# requests: +# cpu: 50m +# memory: 50Mi + ## Node labels for pod assignment + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + # + nodeSelector: {} + + ## Tolerations for pod assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + + ## Affinity for pod assignment (evaluated as template) + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: {} + + ## Use an alternate scheduler, e.g. "stork". + ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ + ## + # schedulerName: "default-scheduler" + +networkPolicy: + ## @param networkPolicy.enabled Enable creation of NetworkPolicy resources. Only Ingress traffic is filtered for now. + ## + enabled: false + ## @param networkPolicy.allowExternal Don't require client label for connections + ## The Policy model to apply. When set to false, only pods with the correct + ## client label will have network access to grafana port defined. + ## When true, grafana will accept connections from any source + ## (with the correct destination port). + ## + ingress: true + ## @param networkPolicy.ingress When true enables the creation + ## an ingress network policy + ## + allowExternal: true + ## @param networkPolicy.explicitNamespacesSelector A Kubernetes LabelSelector to explicitly select namespaces from which traffic could be allowed + ## If explicitNamespacesSelector is missing or set to {}, only client Pods that are in the networkPolicy's namespace + ## and that match other criteria, the ones that have the good label, can reach the grafana. + ## But sometimes, we want the grafana to be accessible to clients from other namespaces, in this case, we can use this + ## LabelSelector to select these namespaces, note that the networkPolicy's namespace should also be explicitly added. + ## + ## Example: + ## explicitNamespacesSelector: + ## matchLabels: + ## role: frontend + ## matchExpressions: + ## - {key: role, operator: In, values: [frontend]} + ## + explicitNamespacesSelector: {} + ## + ## + ## + ## + ## + ## + egress: + ## @param networkPolicy.egress.enabled When enabled, an egress network policy will be + ## created allowing grafana to connect to external data sources from kubernetes cluster. + enabled: false + ## + ## @param networkPolicy.egress.blockDNSResolution When enabled, DNS resolution will be blocked + ## for all pods in the grafana namespace. + blockDNSResolution: false + ## + ## @param networkPolicy.egress.ports Add individual ports to be allowed by the egress + ports: [] + ## Add ports to the egress by specifying - port: + ## E.X. + ## - port: 80 + ## - port: 443 + ## + ## @param networkPolicy.egress.to Allow egress traffic to specific destinations + to: [] + ## Add destinations to the egress by specifying - ipBlock: + ## E.X. + ## to: + ## - namespaceSelector: + ## matchExpressions: + ## - {key: role, operator: In, values: [grafana]} + ## + ## + ## + ## + ## + +# Enable backward compatibility of kubernetes where version below 1.13 doesn't have the enableServiceLinks option +enableKubeBackwardCompatibility: false +useStatefulSet: false +# Create a dynamic manifests via values: +extraObjects: [] + # - apiVersion: "kubernetes-client.io/v1" + # kind: ExternalSecret + # metadata: + # name: grafana-secrets + # spec: + # backendType: gcpSecretsManager + # data: + # - key: grafana-admin-password + # name: adminPassword + +# assertNoLeakedSecrets is a helper function defined in _helpers.tpl that checks if secret +# values are not exposed in the rendered grafana.ini configmap. It is enabled by default. +# +# To pass values into grafana.ini without exposing them in a configmap, use variable expansion: +# https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#variable-expansion +# +# Alternatively, if you wish to allow secret values to be exposed in the rendered grafana.ini configmap, +# you can disable this check by setting assertNoLeakedSecrets to false. +assertNoLeakedSecrets: true diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/Chart.yaml new file mode 100644 index 0000000000..acd648a5b4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: hardenedKubelet +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedKubelet/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/Chart.yaml new file mode 100644 index 0000000000..068932bacb --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: hardenedNodeExporter +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/hardenedNodeExporter/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/Chart.yaml new file mode 100644 index 0000000000..275e02e5dc --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: k3sServer +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/k3sServer/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/Chart.yaml new file mode 100644 index 0000000000..002a6a180d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/Chart.yaml @@ -0,0 +1,32 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/prometheus-community/helm-charts + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-kube-state-metrics +apiVersion: v2 +appVersion: 2.10.1 +description: Install kube-state-metrics to generate and expose cluster-level metrics +home: https://github.com/kubernetes/kube-state-metrics/ +keywords: +- metric +- monitoring +- prometheus +- kubernetes +maintainers: +- email: tariq.ibrahim@mulesoft.com + name: tariq1890 +- email: manuel@rueg.eu + name: mrueg +- email: david@0xdc.me + name: dotdc +name: kube-state-metrics +sources: +- https://github.com/kubernetes/kube-state-metrics/ +type: application +version: 5.16.4 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/README.md new file mode 100644 index 0000000000..843be89e69 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/README.md @@ -0,0 +1,85 @@ +# kube-state-metrics Helm Chart + +Installs the [kube-state-metrics agent](https://github.com/kubernetes/kube-state-metrics). + +## Get Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + + +## Install Chart + +```console +helm install [RELEASE_NAME] prometheus-community/kube-state-metrics [flags] +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-state-metrics [flags] +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### Migrating from stable/kube-state-metrics and kubernetes/kube-state-metrics + +You can upgrade in-place: + +1. [get repository info](#get-repository-info) +1. [upgrade](#upgrading-chart) your existing release name using the new chart repository + +## Upgrading to v3.0.0 + +v3.0.0 includes kube-state-metrics v2.0, see the [changelog](https://github.com/kubernetes/kube-state-metrics/blob/release-2.0/CHANGELOG.md) for major changes on the application-side. + +The upgraded chart now the following changes: + +* Dropped support for helm v2 (helm v3 or later is required) +* collectors key was renamed to resources +* namespace key was renamed to namespaces + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments: + +```console +helm show values prometheus-community/kube-state-metrics +``` + +### kube-rbac-proxy + +You can enable `kube-state-metrics` endpoint protection using `kube-rbac-proxy`. By setting `kubeRBACProxy.enabled: true`, this chart will deploy one RBAC proxy container per endpoint (metrics & telemetry). +To authorize access, authenticate your requests (via a `ServiceAccount` for example) with a `ClusterRole` attached such as: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kube-state-metrics-read +rules: + - apiGroups: [ "" ] + resources: ["services/kube-state-metrics"] + verbs: + - get +``` + +See [kube-rbac-proxy examples](https://github.com/brancz/kube-rbac-proxy/tree/master/examples/resource-attributes) for more details. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/NOTES.txt b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/NOTES.txt new file mode 100644 index 0000000000..3589c24ec3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/NOTES.txt @@ -0,0 +1,23 @@ +kube-state-metrics is a simple service that listens to the Kubernetes API server and generates metrics about the state of the objects. +The exposed metrics can be found here: +https://github.com/kubernetes/kube-state-metrics/blob/master/docs/README.md#exposed-metrics + +The metrics are exported on the HTTP endpoint /metrics on the listening port. +In your case, {{ template "kube-state-metrics.fullname" . }}.{{ template "kube-state-metrics.namespace" . }}.svc.cluster.local:{{ .Values.service.port }}/metrics + +They are served either as plaintext or protobuf depending on the Accept header. +They are designed to be consumed either by Prometheus itself or by a scraper that is compatible with scraping a Prometheus client endpoint. + +{{- if .Values.kubeRBACProxy.enabled}} + +kube-rbac-proxy endpoint protections is enabled: +- Metrics endpoints are now HTTPS +- Ensure that the client authenticates the requests (e.g. via service account) with the following role permissions: +``` +rules: + - apiGroups: [ "" ] + resources: ["services/{{ template "kube-state-metrics.fullname" . }}"] + verbs: + - get +``` +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/_helpers.tpl new file mode 100644 index 0000000000..ed277fbb53 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/_helpers.tpl @@ -0,0 +1,196 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{- define "monitoring_registry" -}} + {{- $temp_registry := (include "system_default_registry" .) -}} + {{- if $temp_registry -}} + {{- trimSuffix "/" $temp_registry -}} + {{- else -}} + {{- .Values.global.imageRegistry -}} + {{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "kube-state-metrics.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "kube-state-metrics.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "kube-state-metrics.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "kube-state-metrics.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "kube-state-metrics.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "kube-state-metrics.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Generate basic labels +*/}} +{{- define "kube-state-metrics.labels" }} +helm.sh/chart: {{ template "kube-state-metrics.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ template "kube-state-metrics.name" . }} +{{- include "kube-state-metrics.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels }} +{{- end }} +{{- if .Values.releaseLabel }} +release: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "kube-state-metrics.selectorLabels" }} +{{- if .Values.selectorOverride }} +{{ toYaml .Values.selectorOverride }} +{{- else }} +app.kubernetes.io/name: {{ include "kube-state-metrics.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end -}} + +{{/* +Formats imagePullSecrets. Input is (dict "Values" .Values "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "kube-state-metrics.imagePullSecrets" -}} +{{- range (concat .Values.global.imagePullSecrets .imagePullSecrets) }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +The image to use for kube-state-metrics +*/}} +{{- define "kube-state-metrics.image" -}} +{{- $registry := (include "monitoring_registry" .) }} +{{- if .Values.image.sha }} +{{- if $registry }} +{{- printf "%s/%s:%s@%s" $registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.sha }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.sha }} +{{- end }} +{{- else }} +{{- if $registry }} +{{- printf "%s/%s:%s" $registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +The image to use for kubeRBACProxy +*/}} +{{- define "kubeRBACProxy.image" -}} +{{- $registry := (include "monitoring_registry" .) }} +{{- if .Values.kubeRBACProxy.image.sha }} +{{- if $registry }} +{{- printf "%s/%s:%s@%s" $registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) .Values.kubeRBACProxy.image.sha }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.kubeRBACProxy.image.registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) .Values.kubeRBACProxy.image.sha }} +{{- end }} +{{- else }} +{{- if $registry }} +{{- printf "%s/%s:%s" $registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.kubeRBACProxy.image.registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml new file mode 100644 index 0000000000..025cd47a88 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.networkPolicy.enabled (eq .Values.networkPolicy.flavor "cilium") }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +spec: + endpointSelector: + matchLabels: + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + egress: + {{- if and .Values.networkPolicy.cilium .Values.networkPolicy.cilium.kubeApiServerSelector }} + {{ toYaml .Values.networkPolicy.cilium.kubeApiServerSelector | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} + ingress: + - toPorts: + - ports: + - port: {{ .Values.service.port | quote }} + protocol: TCP + {{- if .Values.selfMonitor.enabled }} + - port: {{ .Values.selfMonitor.telemetryPort | default 8081 | quote }} + protocol: TCP + {{ end }} +{{ end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..cf9f628d04 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rbac.useClusterRole -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole +{{- if .Values.rbac.useExistingRole }} + name: {{ .Values.rbac.useExistingRole }} +{{- else }} + name: {{ template "kube-state-metrics.fullname" . }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/crs-configmap.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/crs-configmap.yaml new file mode 100644 index 0000000000..d38a75a51d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/crs-configmap.yaml @@ -0,0 +1,16 @@ +{{- if .Values.customResourceState.enabled}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-state-metrics.fullname" . }}-customresourcestate-config + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} +data: + config.yaml: | + {{- toYaml .Values.customResourceState.config | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/deployment.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/deployment.yaml new file mode 100644 index 0000000000..03158eb948 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/deployment.yaml @@ -0,0 +1,314 @@ +apiVersion: apps/v1 +{{- if .Values.autosharding.enabled }} +kind: StatefulSet +{{- else }} +kind: Deployment +{{- end }} +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- if .Values.annotations }} + annotations: +{{ toYaml .Values.annotations | indent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + replicas: {{ .Values.replicas }} + {{- if not .Values.autosharding.enabled }} + strategy: + type: {{ .Values.updateStrategy | default "RollingUpdate" }} + {{- end }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- if .Values.autosharding.enabled }} + serviceName: {{ template "kube-state-metrics.fullname" . }} + volumeClaimTemplates: [] + {{- end }} + template: + metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 8 }} + {{- if .Values.podAnnotations }} + annotations: +{{ toYaml .Values.podAnnotations | indent 8 }} + {{- end }} + spec: + hostNetwork: {{ .Values.hostNetwork }} + serviceAccountName: {{ template "kube-state-metrics.serviceAccountName" . }} + {{- if .Values.securityContext.enabled }} + securityContext: {{- omit .Values.securityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + {{- with .Values.initContainers }} + initContainers: + {{- toYaml . | nindent 6 }} + {{- end }} + containers: + {{- $servicePort := ternary 9090 (.Values.service.port | default 8080) .Values.kubeRBACProxy.enabled}} + {{- $telemetryPort := ternary 9091 (.Values.selfMonitor.telemetryPort | default 8081) .Values.kubeRBACProxy.enabled}} + - name: {{ template "kube-state-metrics.name" . }} + {{- if .Values.autosharding.enabled }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- end }} + args: + {{- if .Values.extraArgs }} + {{- .Values.extraArgs | toYaml | nindent 8 }} + {{- end }} + - --port={{ $servicePort }} + {{- if .Values.collectors }} + - --resources={{ .Values.collectors | join "," }} + {{- end }} + {{- if .Values.metricLabelsAllowlist }} + - --metric-labels-allowlist={{ .Values.metricLabelsAllowlist | join "," }} + {{- end }} + {{- if .Values.metricAnnotationsAllowList }} + - --metric-annotations-allowlist={{ .Values.metricAnnotationsAllowList | join "," }} + {{- end }} + {{- if .Values.metricAllowlist }} + - --metric-allowlist={{ .Values.metricAllowlist | join "," }} + {{- end }} + {{- if .Values.metricDenylist }} + - --metric-denylist={{ .Values.metricDenylist | join "," }} + {{- end }} + {{- $namespaces := list }} + {{- if .Values.namespaces }} + {{- range $ns := join "," .Values.namespaces | split "," }} + {{- $namespaces = append $namespaces (tpl $ns $) }} + {{- end }} + {{- end }} + {{- if .Values.releaseNamespace }} + {{- $namespaces = append $namespaces ( include "kube-state-metrics.namespace" . ) }} + {{- end }} + {{- if $namespaces }} + - --namespaces={{ $namespaces | mustUniq | join "," }} + {{- end }} + {{- if .Values.namespacesDenylist }} + - --namespaces-denylist={{ tpl (.Values.namespacesDenylist | join ",") $ }} + {{- end }} + {{- if .Values.autosharding.enabled }} + - --pod=$(POD_NAME) + - --pod-namespace=$(POD_NAMESPACE) + {{- end }} + {{- if .Values.kubeconfig.enabled }} + - --kubeconfig=/opt/k8s/.kube/config + {{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - --telemetry-host=127.0.0.1 + - --telemetry-port={{ $telemetryPort }} + {{- else }} + {{- if .Values.selfMonitor.telemetryHost }} + - --telemetry-host={{ .Values.selfMonitor.telemetryHost }} + {{- end }} + {{- if .Values.selfMonitor.telemetryPort }} + - --telemetry-port={{ $telemetryPort }} + {{- end }} + {{- end }} + {{- if .Values.customResourceState.enabled }} + - --custom-resource-state-config-file=/etc/customresourcestate/config.yaml + {{- end }} + {{- if or (.Values.kubeconfig.enabled) (.Values.customResourceState.enabled) (.Values.volumeMounts) }} + volumeMounts: + {{- if .Values.kubeconfig.enabled }} + - name: kubeconfig + mountPath: /opt/k8s/.kube/ + readOnly: true + {{- end }} + {{- if .Values.customResourceState.enabled }} + - name: customresourcestate-config + mountPath: /etc/customresourcestate + readOnly: true + {{- end }} + {{- if .Values.volumeMounts }} +{{ toYaml .Values.volumeMounts | indent 8 }} + {{- end }} + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + image: {{ include "kube-state-metrics.image" . }} + {{- if eq .Values.kubeRBACProxy.enabled false }} + ports: + - containerPort: {{ .Values.service.port | default 8080}} + name: "http" + {{- if .Values.selfMonitor.enabled }} + - containerPort: {{ $telemetryPort }} + name: "metrics" + {{- end }} + {{- end }} + livenessProbe: + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + httpGet: + {{- if .Values.hostNetwork }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.livenessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: /healthz + port: {{ $servicePort }} + scheme: {{ upper .Values.livenessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + readinessProbe: + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + httpGet: + {{- if .Values.hostNetwork }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.readinessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ $servicePort }} + scheme: {{ upper .Values.readinessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + {{- if .Values.resources }} + resources: +{{ toYaml .Values.resources | indent 10 }} +{{- end }} +{{- if .Values.containerSecurityContext }} + securityContext: +{{ toYaml .Values.containerSecurityContext | indent 10 }} +{{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - name: kube-rbac-proxy-http + args: + {{- if .Values.kubeRBACProxy.extraArgs }} + {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 8 }} + {{- end }} + - --secure-listen-address=:{{ .Values.service.port | default 8080}} + - --upstream=http://127.0.0.1:{{ $servicePort }}/ + - --proxy-endpoints-port=8888 + - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml + volumeMounts: + - name: kube-rbac-proxy-config + mountPath: /etc/kube-rbac-proxy-config + {{- with .Values.kubeRBACProxy.volumeMounts }} + {{- toYaml . | nindent 10 }} + {{- end }} + imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }} + image: {{ include "kubeRBACProxy.image" . }} + ports: + - containerPort: {{ .Values.service.port | default 8080}} + name: "http" + - containerPort: 8888 + name: "http-healthz" + readinessProbe: + httpGet: + scheme: HTTPS + port: 8888 + path: healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + {{- if .Values.kubeRBACProxy.resources }} + resources: +{{ toYaml .Values.kubeRBACProxy.resources | indent 10 }} +{{- end }} +{{- if .Values.kubeRBACProxy.containerSecurityContext }} + securityContext: +{{ toYaml .Values.kubeRBACProxy.containerSecurityContext | indent 10 }} +{{- end }} + {{- if .Values.selfMonitor.enabled }} + - name: kube-rbac-proxy-telemetry + args: + {{- if .Values.kubeRBACProxy.extraArgs }} + {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 8 }} + {{- end }} + - --secure-listen-address=:{{ .Values.selfMonitor.telemetryPort | default 8081 }} + - --upstream=http://127.0.0.1:{{ $telemetryPort }}/ + - --proxy-endpoints-port=8889 + - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml + volumeMounts: + - name: kube-rbac-proxy-config + mountPath: /etc/kube-rbac-proxy-config + {{- with .Values.kubeRBACProxy.volumeMounts }} + {{- toYaml . | nindent 10 }} + {{- end }} + imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }} + image: {{ include "kubeRBACProxy.image" . }} + ports: + - containerPort: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + name: "metrics" + - containerPort: 8889 + name: "metrics-healthz" + readinessProbe: + httpGet: + scheme: HTTPS + port: 8889 + path: healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + {{- if .Values.kubeRBACProxy.resources }} + resources: +{{ toYaml .Values.kubeRBACProxy.resources | indent 10 }} +{{- end }} +{{- if .Values.kubeRBACProxy.containerSecurityContext }} + securityContext: +{{ toYaml .Values.kubeRBACProxy.containerSecurityContext | indent 10 }} +{{- end }} + {{- end }} + {{- end }} + {{- with .Values.containers }} + {{- toYaml . | nindent 6 }} + {{- end }} +{{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "kube-state-metrics.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.imagePullSecrets) | indent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.nodeSelector }} +{{ toYaml .Values.nodeSelector | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.tolerations }} +{{ toYaml .Values.tolerations | indent 8 }} + {{- end }} + {{- if .Values.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.topologySpreadConstraints | indent 8 }} + {{- end }} + {{- if or (.Values.kubeconfig.enabled) (.Values.customResourceState.enabled) (.Values.volumes) (.Values.kubeRBACProxy.enabled) }} + volumes: + {{- if .Values.kubeconfig.enabled}} + - name: kubeconfig + secret: + secretName: {{ template "kube-state-metrics.fullname" . }}-kubeconfig + {{- end }} + {{- if .Values.kubeRBACProxy.enabled}} + - name: kube-rbac-proxy-config + configMap: + name: {{ template "kube-state-metrics.fullname" . }}-rbac-config + {{- end }} + {{- if .Values.customResourceState.enabled}} + - name: customresourcestate-config + configMap: + name: {{ template "kube-state-metrics.fullname" . }}-customresourcestate-config + {{- end }} + {{- if .Values.volumes }} +{{ toYaml .Values.volumes | indent 8 }} + {{- end }} + {{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/extra-manifests.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/extra-manifests.yaml new file mode 100644 index 0000000000..567f7bf329 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraManifests }} +--- +{{ tpl (toYaml .) $ }} +{{ end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/kubeconfig-secret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/kubeconfig-secret.yaml new file mode 100644 index 0000000000..6af0084502 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/kubeconfig-secret.yaml @@ -0,0 +1,12 @@ +{{- if .Values.kubeconfig.enabled -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-state-metrics.fullname" . }}-kubeconfig + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +type: Opaque +data: + config: '{{ .Values.kubeconfig.secret }}' +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/networkpolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/networkpolicy.yaml new file mode 100644 index 0000000000..309b38ec54 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/networkpolicy.yaml @@ -0,0 +1,43 @@ +{{- if and .Values.networkPolicy.enabled (eq .Values.networkPolicy.flavor "kubernetes") }} +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +spec: + {{- if .Values.networkPolicy.egress }} + ## Deny all egress by default + egress: + {{- toYaml .Values.networkPolicy.egress | nindent 4 }} + {{- end }} + ingress: + {{- if .Values.networkPolicy.ingress }} + {{- toYaml .Values.networkPolicy.ingress | nindent 4 }} + {{- else }} + ## Allow ingress on default ports by default + - ports: + - port: {{ .Values.service.port | default 8080 }} + protocol: TCP + {{- if .Values.selfMonitor.enabled }} + {{- $telemetryPort := ternary 9091 (.Values.selfMonitor.telemetryPort | default 8081) .Values.kubeRBACProxy.enabled}} + - port: {{ $telemetryPort }} + protocol: TCP + {{- end }} + {{- end }} + podSelector: + {{- if .Values.networkPolicy.podSelector }} + {{- toYaml .Values.networkPolicy.podSelector | nindent 4 }} + {{- else }} + matchLabels: + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + {{- end }} + policyTypes: + - Ingress + - Egress +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/pdb.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/pdb.yaml new file mode 100644 index 0000000000..3771b511de --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/pdb.yaml @@ -0,0 +1,18 @@ +{{- if .Values.podDisruptionBudget -}} +{{ if $.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget" -}} +apiVersion: policy/v1 +{{- else -}} +apiVersion: policy/v1beta1 +{{- end }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} +{{ toYaml .Values.podDisruptionBudget | indent 2 }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/podsecuritypolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/podsecuritypolicy.yaml new file mode 100644 index 0000000000..d9d944d740 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/podsecuritypolicy.yaml @@ -0,0 +1,39 @@ +{{- if and .Values.rbac.create (and (or .Values.global.cattle.psp.enabled .Values.podSecurityPolicy.enabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy")) }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +{{- if .Values.podSecurityPolicy.annotations }} + annotations: +{{ toYaml .Values.podSecurityPolicy.annotations | indent 4 }} +{{- end }} +spec: + privileged: false + volumes: + - 'secret' +{{- if .Values.podSecurityPolicy.additionalVolumes }} +{{ toYaml .Values.podSecurityPolicy.additionalVolumes | indent 4 }} +{{- end }} + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrole.yaml new file mode 100644 index 0000000000..c69e01a716 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrole.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.rbac.create (and (or .Values.global.cattle.psp.enabled .Values.podSecurityPolicy.enabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy")) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: psp-{{ template "kube-state-metrics.fullname" . }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-state-metrics.fullname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..df81c49028 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.rbac.create (and (or .Values.global.cattle.psp.enabled .Values.podSecurityPolicy.enabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy")) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: psp-{{ template "kube-state-metrics.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp-{{ template "kube-state-metrics.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/rbac-configmap.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/rbac-configmap.yaml new file mode 100644 index 0000000000..671dc9d660 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/rbac-configmap.yaml @@ -0,0 +1,22 @@ +{{- if .Values.kubeRBACProxy.enabled}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-state-metrics.fullname" . }}-rbac-config + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} +data: + config-file.yaml: |+ + authorization: + resourceAttributes: + namespace: {{ template "kube-state-metrics.namespace" . }} + apiVersion: v1 + resource: services + subresource: {{ template "kube-state-metrics.fullname" . }} + name: {{ template "kube-state-metrics.fullname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/role.yaml new file mode 100644 index 0000000000..0170878376 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/role.yaml @@ -0,0 +1,215 @@ +{{- if not (kindIs "slice" .Values.collectors) }} +{{- fail "Collectors need to be a List since kube-state-metrics chart 3.2.2. Please check README for more information."}} +{{- end }} +{{- if and (eq .Values.rbac.create true) (not .Values.rbac.useExistingRole) -}} +{{- range (ternary (join "," .Values.namespaces | split "," ) (list "") (eq $.Values.rbac.useClusterRole false)) }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +{{- if eq $.Values.rbac.useClusterRole false }} +kind: Role +{{- else }} +kind: ClusterRole +{{- end }} +metadata: + labels: + {{- include "kube-state-metrics.labels" $ | indent 4 }} + name: {{ template "kube-state-metrics.fullname" $ }} +{{- if eq $.Values.rbac.useClusterRole false }} + namespace: {{ . }} +{{- end }} +rules: +{{ if has "certificatesigningrequests" $.Values.collectors }} +- apiGroups: ["certificates.k8s.io"] + resources: + - certificatesigningrequests + verbs: ["list", "watch"] +{{ end -}} +{{ if has "configmaps" $.Values.collectors }} +- apiGroups: [""] + resources: + - configmaps + verbs: ["list", "watch"] +{{ end -}} +{{ if has "cronjobs" $.Values.collectors }} +- apiGroups: ["batch"] + resources: + - cronjobs + verbs: ["list", "watch"] +{{ end -}} +{{ if has "daemonsets" $.Values.collectors }} +- apiGroups: ["extensions", "apps"] + resources: + - daemonsets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "deployments" $.Values.collectors }} +- apiGroups: ["extensions", "apps"] + resources: + - deployments + verbs: ["list", "watch"] +{{ end -}} +{{ if has "endpoints" $.Values.collectors }} +- apiGroups: [""] + resources: + - endpoints + verbs: ["list", "watch"] +{{ end -}} +{{ if has "endpointslices" $.Values.collectors }} +- apiGroups: ["discovery.k8s.io"] + resources: + - endpointslices + verbs: ["list", "watch"] +{{ end -}} +{{ if has "horizontalpodautoscalers" $.Values.collectors }} +- apiGroups: ["autoscaling"] + resources: + - horizontalpodautoscalers + verbs: ["list", "watch"] +{{ end -}} +{{ if has "ingresses" $.Values.collectors }} +- apiGroups: ["extensions", "networking.k8s.io"] + resources: + - ingresses + verbs: ["list", "watch"] +{{ end -}} +{{ if has "jobs" $.Values.collectors }} +- apiGroups: ["batch"] + resources: + - jobs + verbs: ["list", "watch"] +{{ end -}} +{{ if has "leases" $.Values.collectors }} +- apiGroups: ["coordination.k8s.io"] + resources: + - leases + verbs: ["list", "watch"] +{{ end -}} +{{ if has "limitranges" $.Values.collectors }} +- apiGroups: [""] + resources: + - limitranges + verbs: ["list", "watch"] +{{ end -}} +{{ if has "mutatingwebhookconfigurations" $.Values.collectors }} +- apiGroups: ["admissionregistration.k8s.io"] + resources: + - mutatingwebhookconfigurations + verbs: ["list", "watch"] +{{ end -}} +{{ if has "namespaces" $.Values.collectors }} +- apiGroups: [""] + resources: + - namespaces + verbs: ["list", "watch"] +{{ end -}} +{{ if has "networkpolicies" $.Values.collectors }} +- apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: ["list", "watch"] +{{ end -}} +{{ if has "nodes" $.Values.collectors }} +- apiGroups: [""] + resources: + - nodes + verbs: ["list", "watch"] +{{ end -}} +{{ if has "persistentvolumeclaims" $.Values.collectors }} +- apiGroups: [""] + resources: + - persistentvolumeclaims + verbs: ["list", "watch"] +{{ end -}} +{{ if has "persistentvolumes" $.Values.collectors }} +- apiGroups: [""] + resources: + - persistentvolumes + verbs: ["list", "watch"] +{{ end -}} +{{ if has "poddisruptionbudgets" $.Values.collectors }} +- apiGroups: ["policy"] + resources: + - poddisruptionbudgets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "pods" $.Values.collectors }} +- apiGroups: [""] + resources: + - pods + verbs: ["list", "watch"] +{{ end -}} +{{ if has "replicasets" $.Values.collectors }} +- apiGroups: ["extensions", "apps"] + resources: + - replicasets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "replicationcontrollers" $.Values.collectors }} +- apiGroups: [""] + resources: + - replicationcontrollers + verbs: ["list", "watch"] +{{ end -}} +{{ if has "resourcequotas" $.Values.collectors }} +- apiGroups: [""] + resources: + - resourcequotas + verbs: ["list", "watch"] +{{ end -}} +{{ if has "secrets" $.Values.collectors }} +- apiGroups: [""] + resources: + - secrets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "services" $.Values.collectors }} +- apiGroups: [""] + resources: + - services + verbs: ["list", "watch"] +{{ end -}} +{{ if has "statefulsets" $.Values.collectors }} +- apiGroups: ["apps"] + resources: + - statefulsets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "storageclasses" $.Values.collectors }} +- apiGroups: ["storage.k8s.io"] + resources: + - storageclasses + verbs: ["list", "watch"] +{{ end -}} +{{ if has "validatingwebhookconfigurations" $.Values.collectors }} +- apiGroups: ["admissionregistration.k8s.io"] + resources: + - validatingwebhookconfigurations + verbs: ["list", "watch"] +{{ end -}} +{{ if has "volumeattachments" $.Values.collectors }} +- apiGroups: ["storage.k8s.io"] + resources: + - volumeattachments + verbs: ["list", "watch"] +{{ end -}} +{{- if $.Values.kubeRBACProxy.enabled }} +- apiGroups: ["authentication.k8s.io"] + resources: + - tokenreviews + verbs: ["create"] +- apiGroups: ["authorization.k8s.io"] + resources: + - subjectaccessreviews + verbs: ["create"] +{{- end }} +{{- if $.Values.customResourceState.enabled }} +- apiGroups: ["apiextensions.k8s.io"] + resources: + - customresourcedefinitions + verbs: ["list", "watch"] +{{- end }} +{{ if $.Values.rbac.extraRules }} +{{ toYaml $.Values.rbac.extraRules }} +{{ end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/rolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/rolebinding.yaml new file mode 100644 index 0000000000..330651b73f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/rolebinding.yaml @@ -0,0 +1,24 @@ +{{- if and (eq .Values.rbac.create true) (eq .Values.rbac.useClusterRole false) -}} +{{- range (join "," $.Values.namespaces) | split "," }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + {{- include "kube-state-metrics.labels" $ | indent 4 }} + name: {{ template "kube-state-metrics.fullname" $ }} + namespace: {{ . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role +{{- if (not $.Values.rbac.useExistingRole) }} + name: {{ template "kube-state-metrics.fullname" $ }} +{{- else }} + name: {{ $.Values.rbac.useExistingRole }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" $ }} + namespace: {{ template "kube-state-metrics.namespace" $ }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/service.yaml new file mode 100644 index 0000000000..6c486a662a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/service.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + annotations: + {{- if .Values.prometheusScrape }} + prometheus.io/scrape: '{{ .Values.prometheusScrape }}' + {{- end }} + {{- if .Values.service.annotations }} + {{- toYaml .Values.service.annotations | nindent 4 }} + {{- end }} +spec: + type: "{{ .Values.service.type }}" + ports: + - name: "http" + protocol: TCP + port: {{ .Values.service.port | default 8080}} + {{- if .Values.service.nodePort }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: {{ .Values.service.port | default 8080}} + {{ if .Values.selfMonitor.enabled }} + - name: "metrics" + protocol: TCP + port: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + targetPort: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + {{- if .Values.selfMonitor.telemetryNodePort }} + nodePort: {{ .Values.selfMonitor.telemetryNodePort }} + {{- end }} + {{ end }} +{{- if .Values.service.loadBalancerIP }} + loadBalancerIP: "{{ .Values.service.loadBalancerIP }}" +{{- end }} +{{- if .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if .Values.autosharding.enabled }} + clusterIP: None +{{- else if .Values.service.clusterIP }} + clusterIP: "{{ .Values.service.clusterIP }}" +{{- end }} + selector: + {{- include "kube-state-metrics.selectorLabels" . | indent 4 }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/serviceaccount.yaml new file mode 100644 index 0000000000..38a93b31d1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- if .Values.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.serviceAccount.annotations | indent 4 }} +{{- end }} +{{- if or .Values.serviceAccount.imagePullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "kube-state-metrics.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.serviceAccount.imagePullSecrets) | indent 2 }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/servicemonitor.yaml new file mode 100644 index 0000000000..01ec44e067 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/servicemonitor.yaml @@ -0,0 +1,126 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- with .Values.prometheus.monitor.additionalLabels }} + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} + {{- with .Values.prometheus.monitor.annotations }} + annotations: + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }} + {{- with .Values.prometheus.monitor.targetLabels }} + targetLabels: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + {{- with .Values.prometheus.monitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.monitor | indent 2 }} + {{- if .Values.prometheus.monitor.namespaceSelector }} + namespaceSelector: + matchNames: + {{- with .Values.prometheus.monitor.namespaceSelector }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- end }} + selector: + matchLabels: + {{- with .Values.prometheus.monitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + {{- end }} + endpoints: + - port: http + {{- if .Values.prometheus.monitor.interval }} + interval: {{ .Values.prometheus.monitor.interval }} + {{- end }} + {{- if .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.monitor.scrapeTimeout }} + {{- end }} + {{- if .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ .Values.prometheus.monitor.proxyUrl}} + {{- end }} + {{- if .Values.prometheus.monitor.enableHttp2 }} + enableHttp2: {{ .Values.prometheus.monitor.enableHttp2}} + {{- end }} + {{- if .Values.prometheus.monitor.honorLabels }} + honorLabels: true + {{- end }} + metricRelabelings: + {{- if .Values.prometheus.monitor.metricRelabelings }} + {{- toYaml .Values.prometheus.monitor.metricRelabelings | nindent 6 }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml .Values.prometheus.monitor.relabelings | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.scheme }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- end }} + {{- if .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml .Values.prometheus.monitor.tlsConfig | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.monitor.bearerTokenFile }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.selfMonitor.enabled }} + - port: metrics + {{- if .Values.prometheus.monitor.interval }} + interval: {{ .Values.prometheus.monitor.interval }} + {{- end }} + {{- if .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.monitor.scrapeTimeout }} + {{- end }} + {{- if .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ .Values.prometheus.monitor.proxyUrl}} + {{- end }} + {{- if .Values.prometheus.monitor.enableHttp2 }} + enableHttp2: {{ .Values.prometheus.monitor.enableHttp2}} + {{- end }} + {{- if .Values.prometheus.monitor.honorLabels }} + honorLabels: true + {{- end }} + {{- if .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml .Values.prometheus.monitor.relabelings | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.scheme }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- end }} + {{- if .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml .Values.prometheus.monitor.tlsConfig | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.monitor.bearerTokenFile }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-role.yaml new file mode 100644 index 0000000000..489de147c1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-role.yaml @@ -0,0 +1,26 @@ +{{- if and .Values.autosharding.enabled .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - apps + resourceNames: + - {{ template "kube-state-metrics.fullname" . }} + resources: + - statefulsets + verbs: + - get + - list + - watch +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml new file mode 100644 index 0000000000..73b37a4f64 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.autosharding.enabled .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml new file mode 100644 index 0000000000..f46305b517 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml @@ -0,0 +1,44 @@ +{{- if and (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1") (.Values.verticalPodAutoscaler.enabled) }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +spec: + {{- with .Values.verticalPodAutoscaler.recommenders }} + recommenders: + {{- toYaml . | nindent 4 }} + {{- end }} + resourcePolicy: + containerPolicies: + - containerName: {{ template "kube-state-metrics.name" . }} + {{- with .Values.verticalPodAutoscaler.controlledResources }} + controlledResources: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.verticalPodAutoscaler.controlledValues }} + controlledValues: {{ .Values.verticalPodAutoscaler.controlledValues }} + {{- end }} + {{- if .Values.verticalPodAutoscaler.maxAllowed }} + maxAllowed: + {{ toYaml .Values.verticalPodAutoscaler.maxAllowed | nindent 8 }} + {{- end }} + {{- if .Values.verticalPodAutoscaler.minAllowed }} + minAllowed: + {{ toYaml .Values.verticalPodAutoscaler.minAllowed | nindent 8 }} + {{- end }} + targetRef: + apiVersion: apps/v1 + {{- if .Values.autosharding.enabled }} + kind: StatefulSet + {{- else }} + kind: Deployment + {{- end }} + name: {{ template "kube-state-metrics.fullname" . }} + {{- with .Values.verticalPodAutoscaler.updatePolicy }} + updatePolicy: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/values.yaml new file mode 100644 index 0000000000..bc8ee28fda --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kube-state-metrics/values.yaml @@ -0,0 +1,491 @@ +# Default values for kube-state-metrics. +prometheusScrape: true +image: + registry: docker.io + repository: rancher/mirrored-kube-state-metrics-kube-state-metrics + tag: v2.10.1 + sha: "" + pullPolicy: IfNotPresent + +imagePullSecrets: [] +# - name: "image-pull-secret" + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + # + # Allow parent charts to override registry hostname + imageRegistry: "" + +# If set to true, this will deploy kube-state-metrics as a StatefulSet and the data +# will be automatically sharded across <.Values.replicas> pods using the built-in +# autodiscovery feature: https://github.com/kubernetes/kube-state-metrics#automated-sharding +# This is an experimental feature and there are no stability guarantees. +autosharding: + enabled: false + +replicas: 1 + +# Change the deployment strategy when autosharding is disabled. +# ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy +# The default is "RollingUpdate" as per Kubernetes defaults. +# During a release, 'RollingUpdate' can lead to two running instances for a short period of time while 'Recreate' can create a small gap in data. +# updateStrategy: Recreate + +# Number of old history to retain to allow rollback +# Default Kubernetes value is set to 10 +revisionHistoryLimit: 10 + +# List of additional cli arguments to configure kube-state-metrics +# for example: --enable-gzip-encoding, --log-file, etc. +# all the possible args can be found here: https://github.com/kubernetes/kube-state-metrics/blob/master/docs/cli-arguments.md +extraArgs: [] + +service: + port: 8080 + # Default to clusterIP for backward compatibility + type: ClusterIP + nodePort: 0 + loadBalancerIP: "" + # Only allow access to the loadBalancerIP from these IPs + loadBalancerSourceRanges: [] + clusterIP: "" + annotations: {} + +## Additional labels to add to all resources +customLabels: {} + # app: kube-state-metrics + +## Override selector labels +selectorOverride: {} + +## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box +releaseLabel: false + +hostNetwork: false + +rbac: + # If true, create & use RBAC resources + create: true + + # Set to a rolename to use existing role - skipping role creating - but still doing serviceaccount and rolebinding to it, rolename set here. + # useExistingRole: your-existing-role + + # If set to false - Run without Cluteradmin privs needed - ONLY works if namespace is also set (if useExistingRole is set this name is used as ClusterRole or Role to bind to) + useClusterRole: true + + # Add permissions for CustomResources' apiGroups in Role/ClusterRole. Should be used in conjunction with Custom Resource State Metrics configuration + # Example: + # - apiGroups: ["monitoring.coreos.com"] + # resources: ["prometheuses"] + # verbs: ["list", "watch"] + extraRules: [] + +# Configure kube-rbac-proxy. When enabled, creates one kube-rbac-proxy container per exposed HTTP endpoint (metrics and telemetry if enabled). +# The requests are served through the same service but requests are then HTTPS. +kubeRBACProxy: + enabled: false + image: + repository: rancher/mirrored-kube-rbac-proxy + tag: v0.14.0 + sha: "" + pullPolicy: IfNotPresent + + # List of additional cli arguments to configure kube-rbac-prxy + # for example: --tls-cipher-suites, --log-file, etc. + # all the possible args can be found here: https://github.com/brancz/kube-rbac-proxy#usage + extraArgs: [] + + ## Specify security settings for a Container + ## Allows overrides and additional options compared to (Pod) securityContext + ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + containerSecurityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 64Mi + # requests: + # cpu: 10m + # memory: 32Mi + + ## volumeMounts enables mounting custom volumes in rbac-proxy containers + ## Useful for TLS certificates and keys + volumeMounts: [] + # - mountPath: /etc/tls + # name: kube-rbac-proxy-tls + # readOnly: true + +serviceAccount: + # Specifies whether a ServiceAccount should be created, require rbac true + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + # Reference to one or more secrets to be used when pulling images + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + imagePullSecrets: [] + # ServiceAccount annotations. + # Use case: AWS EKS IAM roles for service accounts + # ref: https://docs.aws.amazon.com/eks/latest/userguide/specify-service-account-role.html + annotations: {} + +prometheus: + monitor: + enabled: false + annotations: {} + additionalLabels: {} + namespace: "" + namespaceSelector: [] + jobLabel: "" + targetLabels: [] + podTargetLabels: [] + interval: "" + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + scrapeTimeout: "" + proxyUrl: "" + ## Whether to enable HTTP2 for servicemonitor + # enableHttp2: false + selectorOverride: {} + honorLabels: false + metricRelabelings: [] + relabelings: [] + scheme: "" + ## File to read bearer token for scraping targets + bearerTokenFile: "" + ## Secret to mount to read bearer token for scraping targets. The secret needs + ## to be in the same namespace as the service monitor and accessible by the + ## Prometheus Operator + bearerTokenSecret: {} + # name: secret-name + # key: key-name + tlsConfig: {} + +## Specify if a Pod Security Policy for kube-state-metrics must be created +## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ +## +podSecurityPolicy: + annotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + additionalVolumes: [] + +## Configure network policy for kube-state-metrics +networkPolicy: + enabled: false + # networkPolicy.flavor -- Flavor of the network policy to use. + # Can be: + # * kubernetes for networking.k8s.io/v1/NetworkPolicy + # * cilium for cilium.io/v2/CiliumNetworkPolicy + flavor: kubernetes + + ## Configure the cilium network policy kube-apiserver selector + # cilium: + # kubeApiServerSelector: + # - toEntities: + # - kube-apiserver + + # egress: + # - {} + # ingress: + # - {} + # podSelector: + # matchLabels: + # app.kubernetes.io/name: kube-state-metrics + +securityContext: + enabled: true + runAsGroup: 65534 + runAsUser: 65534 + fsGroup: 65534 + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + +## Specify security settings for a Container +## Allows overrides and additional options compared to (Pod) securityContext +## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container +containerSecurityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + +## Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +nodeSelector: {} + +## Affinity settings for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ +affinity: {} + +## Tolerations for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: [] + +## Topology spread constraints for pod assignment +## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ +topologySpreadConstraints: [] + +# Annotations to be added to the deployment/statefulset +annotations: {} + +# Annotations to be added to the pod +podAnnotations: {} + +## Assign a PriorityClassName to pods if set +# priorityClassName: "" + +# Ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ +podDisruptionBudget: {} + +# Comma-separated list of metrics to be exposed. +# This list comprises of exact metric names and/or regex patterns. +# The allowlist and denylist are mutually exclusive. +metricAllowlist: [] + +# Comma-separated list of metrics not to be enabled. +# This list comprises of exact metric names and/or regex patterns. +# The allowlist and denylist are mutually exclusive. +metricDenylist: [] + +# Comma-separated list of additional Kubernetes label keys that will be used in the resource's +# labels metric. By default the metric contains only name and namespace labels. +# To include additional labels, provide a list of resource names in their plural form and Kubernetes +# label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. +# A single '*' can be provided per resource instead to allow any labels, but that has +# severe performance implications (Example: '=pods=[*]'). +metricLabelsAllowlist: [] + # - namespaces=[k8s-label-1,k8s-label-n] + +# Comma-separated list of Kubernetes annotations keys that will be used in the resource' +# labels metric. By default the metric contains only name and namespace labels. +# To include additional annotations provide a list of resource names in their plural form and Kubernetes +# annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. +# A single '*' can be provided per resource instead to allow any annotations, but that has +# severe performance implications (Example: '=pods=[*]'). +metricAnnotationsAllowList: [] + # - pods=[k8s-annotation-1,k8s-annotation-n] + +# Available collectors for kube-state-metrics. +# By default, all available resources are enabled, comment out to disable. +collectors: + - certificatesigningrequests + - configmaps + - cronjobs + - daemonsets + - deployments + - endpoints + - horizontalpodautoscalers + - ingresses + - jobs + - leases + - limitranges + - mutatingwebhookconfigurations + - namespaces + - networkpolicies + - nodes + - persistentvolumeclaims + - persistentvolumes + - poddisruptionbudgets + - pods + - replicasets + - replicationcontrollers + - resourcequotas + - secrets + - services + - statefulsets + - storageclasses + - validatingwebhookconfigurations + - volumeattachments + +# Enabling kubeconfig will pass the --kubeconfig argument to the container +kubeconfig: + enabled: false + # base64 encoded kube-config file + secret: + +# Enabling support for customResourceState, will create a configMap including your config that will be read from kube-state-metrics +customResourceState: + enabled: false + # Add (Cluster)Role permissions to list/watch the customResources defined in the config to rbac.extraRules + config: {} + +# Enable only the release namespace for collecting resources. By default all namespaces are collected. +# If releaseNamespace and namespaces are both set a merged list will be collected. +releaseNamespace: false + +# Comma-separated list(string) or yaml list of namespaces to be enabled for collecting resources. By default all namespaces are collected. +namespaces: "" + +# Comma-separated list of namespaces not to be enabled. If namespaces and namespaces-denylist are both set, +# only namespaces that are excluded in namespaces-denylist will be used. +namespacesDenylist: "" + +## Override the deployment namespace +## +namespaceOverride: "" + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 64Mi + # requests: + # cpu: 10m + # memory: 32Mi + +## Provide a k8s version to define apiGroups for podSecurityPolicy Cluster Role. +## For example: kubeTargetVersionOverride: 1.14.9 +## +kubeTargetVersionOverride: "" + +# Enable self metrics configuration for service and Service Monitor +# Default values for telemetry configuration can be overridden +# If you set telemetryNodePort, you must also set service.type to NodePort +selfMonitor: + enabled: false + # telemetryHost: 0.0.0.0 + # telemetryPort: 8081 + # telemetryNodePort: 0 + +# Enable vertical pod autoscaler support for kube-state-metrics +verticalPodAutoscaler: + enabled: false + + # Recommender responsible for generating recommendation for the object. + # List should be empty (then the default recommender will generate the recommendation) + # or contain exactly one recommender. + # recommenders: [] + # - name: custom-recommender-performance + + # List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory + controlledResources: [] + # Specifies which resource values should be controlled: RequestsOnly or RequestsAndLimits. + # controlledValues: RequestsAndLimits + + # Define the max allowed resources for the pod + maxAllowed: {} + # cpu: 200m + # memory: 100Mi + # Define the min allowed resources for the pod + minAllowed: {} + # cpu: 200m + # memory: 100Mi + + # updatePolicy: + # Specifies minimal number of replicas which need to be alive for VPA Updater to attempt pod eviction + # minReplicas: 1 + # Specifies whether recommended updates are applied when a Pod is started and whether recommended updates + # are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto". + # updateMode: Auto + +# volumeMounts are used to add custom volume mounts to deployment. +# See example below +volumeMounts: [] +# - mountPath: /etc/config +# name: config-volume + +# volumes are used to add custom volumes to deployment +# See example below +volumes: [] +# - configMap: +# name: cm-for-volume +# name: config-volume + +# Extra manifests to deploy as an array +extraManifests: [] + # - apiVersion: v1 + # kind: ConfigMap + # metadata: + # labels: + # name: prometheus-extra + # data: + # extra-data: "value" + +## Containers allows injecting additional containers. +containers: [] + # - name: crd-init + # image: kiwigrid/k8s-sidecar:latest + +## InitContainers allows injecting additional initContainers. +initContainers: [] + # - name: crd-sidecar + # image: kiwigrid/k8s-sidecar:latest + +## Liveness probe +## +livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + +## Readiness probe +## +readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/Chart.yaml new file mode 100644 index 0000000000..ad9fba247a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: kubeAdmControllerManager +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmControllerManager/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/Chart.yaml new file mode 100644 index 0000000000..d144d3ee2a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: kubeAdmEtcd +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmEtcd/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/Chart.yaml new file mode 100644 index 0000000000..a1222c4feb --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: kubeAdmProxy +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmProxy/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/Chart.yaml new file mode 100644 index 0000000000..78a44159a9 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: kubeAdmScheduler +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/kubeAdmScheduler/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/Chart.yaml new file mode 100644 index 0000000000..d067725a17 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/Chart.yaml @@ -0,0 +1,28 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-prometheus-adapter +apiVersion: v1 +appVersion: v0.10.0 +description: A Helm chart for k8s prometheus adapter +home: https://github.com/kubernetes-sigs/prometheus-adapter +keywords: +- hpa +- metrics +- prometheus +- adapter +kubeVersion: '>=1.26.0-0' +maintainers: +- email: mattias.gees@jetstack.io + name: mattiasgees +- name: steven-sheehy +- email: hfernandez@mesosphere.com + name: hectorj2f +name: prometheus-adapter +sources: +- https://github.com/kubernetes/charts +- https://github.com/kubernetes-sigs/prometheus-adapter +version: 4.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/README.md new file mode 100644 index 0000000000..d77bb0c920 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/README.md @@ -0,0 +1,160 @@ +# Prometheus Adapter + +Installs the [Prometheus Adapter](https://github.com/kubernetes-sigs/prometheus-adapter) for the Custom Metrics API. Custom metrics are used in Kubernetes by [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) to scale workloads based upon your own metric pulled from an external metrics provider like Prometheus. This chart complements the [metrics-server](https://github.com/helm/charts/tree/master/stable/metrics-server) chart that provides resource only metrics. + +## Prerequisites + +Kubernetes 1.14+ + +## Get Helm Repositories Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Helm Chart + +```console +helm install [RELEASE_NAME] prometheus-community/prometheus-adapter +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Helm Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Helm Chart + +```console +helm upgrade [RELEASE_NAME] [CHART] --install +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### To 4.2.0 + +Readiness and liveness probes are now fully configurable through values `readinessProbe` and `livenessProbe`. The previous values have been kept as defaults. + +### To 4.0.0 + +Previously, security context of the container was set directly in the deployment template. This release makes it configurable through the new configuration variable `securityContext` whilst keeping the previously set values as defaults. Furthermore, previous variable `runAsUser` is now set in `securityContext` and is not used any longer. Please, use `securityContext.runAsUser` instead. In the same security context, `seccompProfile` has been enabled and set to type `RuntimeDefault`. + +### To 3.0.0 + +Due to a change in deployment labels, the upgrade requires `helm upgrade --force` in order to re-create the deployment. + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values prometheus-community/prometheus-adapter +``` + +### Prometheus Service Endpoint + +To use the chart, ensure the `prometheus.url` and `prometheus.port` are configured with the correct Prometheus service endpoint. If Prometheus is exposed under HTTPS the host's CA Bundle must be exposed to the container using `extraVolumes` and `extraVolumeMounts`. + +### Adapter Rules + +Additionally, the chart comes with a set of default rules out of the box but they may pull in too many metrics or not map them correctly for your needs. Therefore, it is recommended to populate `rules.custom` with a list of rules (see the [config document](https://github.com/kubernetes-sigs/prometheus-adapter/blob/master/docs/config.md) for the proper format). + +### Horizontal Pod Autoscaler Metrics + +Finally, to configure your Horizontal Pod Autoscaler to use the custom metric, see the custom metrics section of the [HPA walkthrough](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics). + +The Prometheus Adapter can serve three different [metrics APIs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-metrics-apis): + +### Custom Metrics + +Enabling this option will cause custom metrics to be served at `/apis/custom.metrics.k8s.io/v1beta1`. Enabled by default when `rules.default` is true, but can be customized by populating `rules.custom`: + +```yaml +rules: + custom: + - seriesQuery: '{__name__=~"^some_metric_count$"}' + resources: + template: <<.Resource>> + name: + matches: "" + as: "my_custom_metric" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) +``` + +### External Metrics + +Enabling this option will cause external metrics to be served at `/apis/external.metrics.k8s.io/v1beta1`. Can be enabled by populating `rules.external`: + +```yaml +rules: + external: + - seriesQuery: '{__name__=~"^some_metric_count$"}' + resources: + template: <<.Resource>> + name: + matches: "" + as: "my_external_metric" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) +``` + +### Resource Metrics + +Enabling this option will cause resource metrics to be served at `/apis/metrics.k8s.io/v1beta1`. Resource metrics will allow pod CPU and Memory metrics to be used in [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) as well as the `kubectl top` command. Can be enabled by populating `rules.resource`: + +```yaml +rules: + resource: + cpu: + containerQuery: | + sum by (<<.GroupBy>>) ( + rate(container_cpu_usage_seconds_total{container!="",<<.LabelMatchers>>}[3m]) + ) + nodeQuery: | + sum by (<<.GroupBy>>) ( + rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",<<.LabelMatchers>>}[3m]) + ) + resources: + overrides: + node: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container + memory: + containerQuery: | + sum by (<<.GroupBy>>) ( + avg_over_time(container_memory_working_set_bytes{container!="",<<.LabelMatchers>>}[3m]) + ) + nodeQuery: | + sum by (<<.GroupBy>>) ( + avg_over_time(node_memory_MemTotal_bytes{<<.LabelMatchers>>}[3m]) + - + avg_over_time(node_memory_MemAvailable_bytes{<<.LabelMatchers>>}[3m]) + ) + resources: + overrides: + node: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container + window: 3m +``` + +**NOTE:** Setting a value for `rules.resource` will also deploy the resource metrics API service, providing the same functionality as [metrics-server](https://github.com/helm/charts/tree/master/stable/metrics-server). As such it is not possible to deploy them both in the same cluster. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/NOTES.txt b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/NOTES.txt new file mode 100644 index 0000000000..b7b9b99322 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/NOTES.txt @@ -0,0 +1,9 @@ +{{ template "k8s-prometheus-adapter.fullname" . }} has been deployed. +In a few minutes you should be able to list metrics using the following command(s): +{{ if .Values.rules.resource }} + kubectl get --raw /apis/metrics.k8s.io/v1beta1 +{{- end }} + kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1 +{{ if .Values.rules.external }} + kubectl get --raw /apis/external.metrics.k8s.io/v1beta1 +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/_helpers.tpl new file mode 100644 index 0000000000..edbb829b2b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/_helpers.tpl @@ -0,0 +1,113 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "k8s-prometheus-adapter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "k8s-prometheus-adapter.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "k8s-prometheus-adapter.namespace" -}} +{{- default .Release.Namespace .Values.namespaceOverride -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "k8s-prometheus-adapter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Generate basic labels +*/}} +{{- define "k8s-prometheus-adapter.labels" }} +helm.sh/chart: {{ include "k8s-prometheus-adapter.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ template "k8s-prometheus-adapter.name" . }} +{{- include "k8s-prometheus-adapter.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "k8s-prometheus-adapter.selectorLabels" }} +app.kubernetes.io/name: {{ include "k8s-prometheus-adapter.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "k8s-prometheus-adapter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "k8s-prometheus-adapter.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Get Policy API Version */}} +{{- define "k8s-prometheus-adapter.pdb.apiVersion" -}} +{{- if and (.Capabilities.APIVersions.Has "policy/v1") (semverCompare ">= 1.21-0" .Capabilities.KubeVersion.Version) -}} + {{- print "policy/v1" -}} +{{- else -}} + {{- print "policy/v1beta1" -}} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/certmanager.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/certmanager.yaml new file mode 100644 index 0000000000..4e32c964c6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/certmanager.yaml @@ -0,0 +1,76 @@ +{{- if .Values.certManager.enabled -}} +--- +# Create a selfsigned Issuer, in order to create a root CA certificate for +# signing webhook serving certificates +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-self-signed-issuer + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + selfSigned: {} +--- +# Generate a CA Certificate used to sign certificates for the webhook +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-root-cert + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }}-root-cert + duration: {{ .Values.certManager.caCertDuration }} + issuerRef: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-self-signed-issuer + commonName: "ca.webhook.prometheus-adapter" + isCA: true +--- +# Create an Issuer that uses the above generated CA certificate to issue certs +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-root-issuer + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + ca: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }}-root-cert +--- +# Finally, generate a serving certificate for the apiservices to use +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-cert + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }} + duration: {{ .Values.certManager.certDuration }} + issuerRef: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-root-issuer + dnsNames: + - {{ template "k8s-prometheus-adapter.fullname" . }} + - {{ template "k8s-prometheus-adapter.fullname" . }}.{{ include "k8s-prometheus-adapter.namespace" . }} + - {{ template "k8s-prometheus-adapter.fullname" . }}.{{ include "k8s-prometheus-adapter.namespace" . }}.svc +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml new file mode 100644 index 0000000000..6701e6ba08 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-system-auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml new file mode 100644 index 0000000000..67efd2aa2f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml new file mode 100644 index 0000000000..2c690a03cc --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml @@ -0,0 +1,24 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +rules: +- apiGroups: + - "" + resources: + - namespaces + - pods + - services + - configmaps + verbs: + - get + - list + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/configmap.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/configmap.yaml new file mode 100644 index 0000000000..17f415d970 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/configmap.yaml @@ -0,0 +1,97 @@ +{{- if not .Values.rules.existing -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +data: + config.yaml: | +{{- if or .Values.rules.default .Values.rules.custom }} + rules: +{{- if .Values.rules.default }} + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: [] + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: + - isNot: ^container_.*_seconds_total$ + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: + - isNot: ^container_.*_total$ + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)$ + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>,container!="POD"}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_total$ + resources: + template: <<.Resource>> + name: + matches: "" + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_seconds_total + resources: + template: <<.Resource>> + name: + matches: ^(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: [] + resources: + template: <<.Resource>> + name: + matches: ^(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) +{{- end -}} +{{- if .Values.rules.custom }} +{{ toYaml .Values.rules.custom | indent 4 }} +{{- end -}} +{{- end -}} +{{- if .Values.rules.external }} + externalRules: +{{ toYaml .Values.rules.external | indent 4 }} +{{- end -}} +{{- if .Values.rules.resource }} + resourceRules: +{{ toYaml .Values.rules.resource | indent 6 }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml new file mode 100644 index 0000000000..8b7b4e555e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml @@ -0,0 +1,34 @@ +{{- if or .Values.rules.default .Values.rules.custom }} +{{- if .Capabilities.APIVersions.Has "apiregistration.k8s.io/v1" }} +apiVersion: apiregistration.k8s.io/v1 +{{- else }} +apiVersion: apiregistration.k8s.io/v1beta1 +{{- end }} +kind: APIService +metadata: +{{- if or .Values.certManager.enabled .Values.customAnnotations }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: v1beta1.custom.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} + {{- if .Values.tls.enable }} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: custom.metrics.k8s.io + version: v1beta1 + {{- if not (or .Values.tls.enable .Values.certManager.enabled) }} + insecureSkipTLSVerify: true + {{- end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml new file mode 100644 index 0000000000..0cc6920836 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml @@ -0,0 +1,24 @@ +{{- /* +This if must be aligned with custom-metrics-cluster-role.yaml +as otherwise this binding will point to not existing role. +*/ -}} +{{- if and .Values.rbac.create (or .Values.rules.default .Values.rules.custom) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-server-resources +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml new file mode 100644 index 0000000000..4aa15ffe99 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.rbac.create (or .Values.rules.default .Values.rules.custom) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-server-resources +rules: +- apiGroups: + - custom.metrics.k8s.io + resources: ["*"] + verbs: ["*"] +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/deployment.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/deployment.yaml new file mode 100644 index 0000000000..a7ea3310a0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/deployment.yaml @@ -0,0 +1,143 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + {{- if or .Values.customAnnotations .Values.deploymentAnnotations }} + annotations: + {{- with .Values.customAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.deploymentAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +spec: + replicas: {{ .Values.replicas }} + strategy: {{ toYaml .Values.strategy | nindent 4 }} + selector: + matchLabels: + {{- include "k8s-prometheus-adapter.selectorLabels" . | indent 6 }} + template: + metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + name: {{ template "k8s-prometheus-adapter.name" . }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.customAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + {{- if .Values.hostNetwork.enabled }} + hostNetwork: true + {{- end }} + {{- if .Values.dnsPolicy }} + dnsPolicy: {{ .Values.dnsPolicy }} + {{- end}} + containers: + - name: {{ .Chart.Name }} + image: "{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 8 }} + {{- end }} + args: + - /adapter + - --secure-port={{ .Values.listenPort }} + {{- if or .Values.tls.enable .Values.certManager.enabled }} + - --tls-cert-file=/var/run/serving-cert/tls.crt + - --tls-private-key-file=/var/run/serving-cert/tls.key + {{- end }} + - --cert-dir=/tmp/cert + - --prometheus-url={{ tpl .Values.prometheus.url . }}{{ if .Values.prometheus.port }}:{{ .Values.prometheus.port }}{{end}}{{ .Values.prometheus.path }} + - --metrics-relist-interval={{ .Values.metricsRelistInterval }} + - --v={{ .Values.logLevel }} + - --config=/etc/adapter/config.yaml + {{- if .Values.extraArguments }} + {{- toYaml .Values.extraArguments | trim | nindent 8 }} + {{- end }} + ports: + - containerPort: {{ .Values.listenPort }} + name: https + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- if .Values.resources }} + resources: + {{- toYaml .Values.resources | nindent 10 }} + {{- end }} + {{- with .Values.dnsConfig }} + dnsConfig: + {{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} + volumeMounts: + {{- if .Values.extraVolumeMounts }} + {{ toYaml .Values.extraVolumeMounts | trim | nindent 8 }} + {{ end }} + - mountPath: /etc/adapter/ + name: config + readOnly: true + - mountPath: /tmp + name: tmp + {{- if or .Values.tls.enable .Values.certManager.enabled }} + - mountPath: /var/run/serving-cert + name: volume-serving-cert + readOnly: true + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.nodeSelector }} +{{- toYaml .Values.nodeSelector | nindent 8 }} +{{- end }} + affinity: + {{- toYaml .Values.affinity | nindent 8 }} + topologySpreadConstraints: + {{- toYaml .Values.topologySpreadConstraints | nindent 8 }} + priorityClassName: {{ .Values.priorityClassName }} + {{- if .Values.podSecurityContext }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.tolerations }} +{{- toYaml .Values.tolerations | nindent 8 }} +{{- end }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + {{- if .Values.extraVolumes }} + {{ toYaml .Values.extraVolumes | trim | nindent 6 }} + {{ end }} + - name: config + configMap: + name: {{ .Values.rules.existing | default (include "k8s-prometheus-adapter.fullname" . ) }} + - name: tmp + emptyDir: {} + {{- if or .Values.tls.enable .Values.certManager.enabled }} + - name: volume-serving-cert + secret: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }} + {{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml new file mode 100644 index 0000000000..21339af128 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml @@ -0,0 +1,34 @@ +{{- if .Values.rules.external }} +{{- if .Capabilities.APIVersions.Has "apiregistration.k8s.io/v1" }} +apiVersion: apiregistration.k8s.io/v1 +{{- else }} +apiVersion: apiregistration.k8s.io/v1beta1 +{{- end }} +kind: APIService +metadata: +{{- if or .Values.certManager.enabled .Values.customAnnotations }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: v1beta1.external.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} + {{- if .Values.tls.enable }} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: external.metrics.k8s.io + version: v1beta1 + {{- if not (or .Values.tls.enable .Values.certManager.enabled) }} + insecureSkipTLSVerify: true + {{- end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml new file mode 100644 index 0000000000..05547bd323 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rules.external -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller-external-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-external-metrics +subjects: +- kind: ServiceAccount + name: horizontal-pod-autoscaler + namespace: kube-system +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml new file mode 100644 index 0000000000..212ea78b25 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.rbac.create .Values.rules.external -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-external-metrics +rules: +- apiGroups: + - "external.metrics.k8s.io" + resources: + - "*" + verbs: + - list + - get + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/pdb.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/pdb.yaml new file mode 100644 index 0000000000..205761a9f1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/pdb.yaml @@ -0,0 +1,23 @@ +{{- if .Values.podDisruptionBudget.enabled }} +apiVersion: {{ include "k8s-prometheus-adapter.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + {{- if .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + {{- include "k8s-prometheus-adapter.selectorLabels" . | indent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/psp.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/psp.yaml new file mode 100644 index 0000000000..fded5a7491 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/psp.yaml @@ -0,0 +1,66 @@ +{{- if and (or .Values.global.cattle.psp.enabled .Values.psp.create) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + {{- if .Values.hostNetwork.enabled }} + hostNetwork: true + hostPorts: + - min: {{ .Values.listenPort }} + max: {{ .Values.listenPort }} + {{- end }} + fsGroup: + rule: RunAsAny + runAsGroup: + rule: RunAsAny + runAsUser: + rule: MustRunAs + ranges: + - min: 1024 + max: 65535 + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - secret + - emptyDir + - configMap +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +rules: +- apiGroups: + - 'policy' + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "k8s-prometheus-adapter.fullname" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml new file mode 100644 index 0000000000..0cc9fff6a2 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml @@ -0,0 +1,34 @@ +{{- if .Values.rules.resource}} +{{- if .Capabilities.APIVersions.Has "apiregistration.k8s.io/v1" }} +apiVersion: apiregistration.k8s.io/v1 +{{- else }} +apiVersion: apiregistration.k8s.io/v1beta1 +{{- end }} +kind: APIService +metadata: +{{- if or .Values.certManager.enabled .Values.customAnnotations }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: v1beta1.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} + {{- if .Values.tls.enable }} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: metrics.k8s.io + version: v1beta1 + {{- if not (or .Values.tls.enable .Values.certManager.enabled) }} + insecureSkipTLSVerify: true + {{- end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml new file mode 100644 index 0000000000..3c247e48d2 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rules.resource -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-metrics +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml new file mode 100644 index 0000000000..73d8953046 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.rbac.create .Values.rules.resource -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-metrics +rules: +- apiGroups: + - "" + resources: + - pods + - nodes + - nodes/stats + verbs: + - get + - list + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml new file mode 100644 index 0000000000..d3c77c1c65 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml @@ -0,0 +1,21 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/secret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/secret.yaml new file mode 100644 index 0000000000..3e7e8887bd --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/secret.yaml @@ -0,0 +1,17 @@ +{{- if .Values.tls.enable -}} +apiVersion: v1 +kind: Secret +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +type: kubernetes.io/tls +data: + tls.crt: {{ b64enc .Values.tls.certificate }} + tls.key: {{ b64enc .Values.tls.key }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/service.yaml new file mode 100644 index 0000000000..ddac37cfa1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/service.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: Service +metadata: + {{- if or .Values.service.annotations .Values.customAnnotations }} + annotations: + {{- if .Values.service.annotations }} + {{ toYaml .Values.service.annotations | indent 4 }} + {{- end }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +spec: + ports: + - port: {{ .Values.service.port }} + protocol: TCP + targetPort: https + selector: + {{- include "k8s-prometheus-adapter.selectorLabels" . | indent 4 }} + type: {{ .Values.service.type }} + {{- if .Values.service.clusterIP }} + clusterIP: {{ .Values.service.clusterIP }} + {{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/serviceaccount.yaml new file mode 100644 index 0000000000..30a169ae0e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/templates/serviceaccount.yaml @@ -0,0 +1,18 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +{{- if or .Values.serviceAccount.annotations .Values.customAnnotations }} + annotations: + {{- if .Values.serviceAccount.annotations }} + {{- toYaml .Values.serviceAccount.annotations | nindent 4 }} + {{- end }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/values.yaml new file mode 100644 index 0000000000..a1445a23f1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-adapter/values.yaml @@ -0,0 +1,277 @@ +# Default values for k8s-prometheus-adapter.. +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + +affinity: {} + +topologySpreadConstraints: [] + +image: + repository: rancher/mirrored-prometheus-adapter-prometheus-adapter + tag: v0.12.0 + pullPolicy: IfNotPresent + +logLevel: 4 + +metricsRelistInterval: 1m + +listenPort: 6443 + +nodeSelector: {} + +priorityClassName: "" + +## Override the release namespace (for multi-namespace deployments in combined charts) +namespaceOverride: "" + +## Additional annotations to add to all resources +customAnnotations: {} + # role: custom-metrics + +## Additional labels to add to all resources +customLabels: {} + # monitoring: prometheus-adapter + +# Url to access prometheus +prometheus: + # Value is templated + url: http://prometheus.default.svc + port: 9090 + path: "" + +replicas: 1 + +# k8s 1.21 needs fsGroup to be set for non root deployments +# ref: https://github.com/kubernetes/kubernetes/issues/70679 +podSecurityContext: + fsGroup: 10001 + +# SecurityContext of the container +# ref. https://kubernetes.io/docs/tasks/configure-pod-container/security-context +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["all"] + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10001 + seccompProfile: + type: RuntimeDefault + +rbac: + # Specifies whether RBAC resources should be created + create: true + +psp: + # Specifies whether PSP resources should be created + create: false + +serviceAccount: + # Specifies whether a service account should be created + create: true + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: + # ServiceAccount annotations. + # Use case: AWS EKS IAM roles for service accounts + # ref: https://docs.aws.amazon.com/eks/latest/userguide/specify-service-account-role.html + annotations: {} + +# Custom DNS configuration to be added to prometheus-adapter pods +dnsConfig: {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots + # value: "2" + # - name: edns0 + +resources: {} + # requests: + # cpu: 100m + # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + +# Configure liveness probe +# https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#Probe +livenessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + timeoutSeconds: 5 + +# Configure readiness probe +readinessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + timeoutSeconds: 5 + +rules: + default: true + + custom: [] + # - seriesQuery: '{__name__=~"^some_metric_count$"}' + # resources: + # template: <<.Resource>> + # name: + # matches: "" + # as: "my_custom_metric" + # metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + + # Mounts a configMap with pre-generated rules for use. Overrides the + # default, custom, external and resource entries + existing: + + external: [] + # - seriesQuery: '{__name__=~"^some_metric_count$"}' + # resources: + # template: <<.Resource>> + # name: + # matches: "" + # as: "my_external_metric" + # metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + + # resource: + # cpu: + # containerQuery: | + # sum by (<<.GroupBy>>) ( + # rate(container_cpu_usage_seconds_total{container!="",<<.LabelMatchers>>}[3m]) + # ) + # nodeQuery: | + # sum by (<<.GroupBy>>) ( + # rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",<<.LabelMatchers>>}[3m]) + # ) + # resources: + # overrides: + # node: + # resource: node + # namespace: + # resource: namespace + # pod: + # resource: pod + # containerLabel: container + # memory: + # containerQuery: | + # sum by (<<.GroupBy>>) ( + # avg_over_time(container_memory_working_set_bytes{container!="",<<.LabelMatchers>>}[3m]) + # ) + # nodeQuery: | + # sum by (<<.GroupBy>>) ( + # avg_over_time(node_memory_MemTotal_bytes{<<.LabelMatchers>>}[3m]) + # - + # avg_over_time(node_memory_MemAvailable_bytes{<<.LabelMatchers>>}[3m]) + # ) + # resources: + # overrides: + # node: + # resource: node + # namespace: + # resource: namespace + # pod: + # resource: pod + # containerLabel: container + # window: 3m + +service: + annotations: {} + port: 443 + type: ClusterIP + # clusterIP: 1.2.3.4 + +tls: + enable: false + ca: |- + # Public CA file that signed the APIService + key: |- + # Private key of the APIService + certificate: |- + # Public key of the APIService + +# Set environment variables from secrets, configmaps or by setting them as name/value +env: [] + # - name: TMP_DIR + # value: /tmp + # - name: PASSWORD + # valueFrom: + # secretKeyRef: + # name: mysecret + # key: password + # optional: false + +# Any extra arguments +extraArguments: [] + # - --tls-private-key-file=/etc/tls/tls.key + # - --tls-cert-file=/etc/tls/tls.crt + +# Any extra volumes +extraVolumes: [] + # - name: example-name + # hostPath: + # path: /path/on/host + # type: DirectoryOrCreate + # - name: ssl-certs + # hostPath: + # path: /etc/ssl/certs/ca-bundle.crt + # type: File + +# Any extra volume mounts +extraVolumeMounts: [] + # - name: example-name + # mountPath: /path/in/container + # - name: ssl-certs + # mountPath: /etc/ssl/certs/ca-certificates.crt + # readOnly: true + +tolerations: [] + +# Labels added to the pod +podLabels: {} + +# Annotations added to the pod +podAnnotations: {} + +# Annotations added to the deployment +deploymentAnnotations: {} + +hostNetwork: + # Specifies if prometheus-adapter should be started in hostNetwork mode. + # + # You would require this enabled if you use alternate overlay networking for pods and + # API server unable to communicate with metrics-server. As an example, this is required + # if you use Weave network on EKS. See also dnsPolicy + enabled: false + +# When hostNetwork is enabled, you probably want to set this to ClusterFirstWithHostNet +# dnsPolicy: ClusterFirstWithHostNet + +# Deployment strategy type +strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 25% + maxSurge: 25% + +podDisruptionBudget: + # Specifies if PodDisruptionBudget should be enabled + # When enabled, minAvailable or maxUnavailable should also be defined. + enabled: false + minAvailable: + maxUnavailable: 1 + +certManager: + enabled: false + caCertDuration: 43800h + certDuration: 8760h diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/Chart.yaml new file mode 100644 index 0000000000..9130cbcc91 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/Chart.yaml @@ -0,0 +1,25 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: |- + - name: Chart Source + url: https://github.com/prometheus-community/helm-charts +apiVersion: v2 +appVersion: 1.7.0 +description: A Helm chart for prometheus node-exporter +home: https://github.com/prometheus/node_exporter/ +keywords: +- node-exporter +- prometheus +- exporter +maintainers: +- email: gianrubio@gmail.com + name: gianrubio +- email: zanhsieh@gmail.com + name: zanhsieh +- email: rootsandtrees@posteo.de + name: zeritti +name: prometheus-node-exporter +sources: +- https://github.com/prometheus/node_exporter/ +type: application +version: 4.30.3 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/README.md new file mode 100644 index 0000000000..149b982267 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/README.md @@ -0,0 +1,97 @@ + +# Prometheus Node Exporter + +Prometheus exporter for hardware and OS metrics exposed by *NIX kernels, written in Go with pluggable metric collectors. + +This chart bootstraps a Prometheus [Node Exporter](http://github.com/prometheus/node_exporter) daemonset on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Get Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Chart + +```console +helm install [RELEASE_NAME] prometheus-community/prometheus-node-exporter +``` + +_See [configuration](#configuring) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] prometheus-community/prometheus-node-exporter --install +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### 3.x to 4.x + +Starting from version 4.0.0, the `node exporter` chart is using the [Kubernetes recommended labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/). Therefore you have to delete the daemonset before you upgrade. + +```console +kubectl delete daemonset -l app=prometheus-node-exporter +helm upgrade -i prometheus-node-exporter prometheus-community/prometheus-node-exporter +``` + +If you use your own custom [ServiceMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor) or [PodMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#podmonitor), please ensure to upgrade their `selector` fields accordingly to the new labels. + +### From 2.x to 3.x + +Change the following: + +```yaml +hostRootFsMount: true +``` + +to: + +```yaml +hostRootFsMount: + enabled: true + mountPropagation: HostToContainer +``` + +## Configuring + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values prometheus-community/prometheus-node-exporter +``` + +### kube-rbac-proxy + +You can enable `prometheus-node-exporter` endpoint protection using `kube-rbac-proxy`. By setting `kubeRBACProxy.enabled: true`, this chart will deploy a RBAC proxy container protecting the node-exporter endpoint. +To authorize access, authenticate your requests (via a `ServiceAccount` for example) with a `ClusterRole` attached such as: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus-node-exporter-read +rules: + - apiGroups: [ "" ] + resources: ["services/node-exporter-prometheus-node-exporter"] + verbs: + - get +``` + +See [kube-rbac-proxy examples](https://github.com/brancz/kube-rbac-proxy/tree/master/examples/resource-attributes) for more details. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/NOTES.txt b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/NOTES.txt new file mode 100644 index 0000000000..8c5391f1f7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/NOTES.txt @@ -0,0 +1,29 @@ +1. Get the application URL by running these commands: +{{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ template "prometheus-node-exporter.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "prometheus-node-exporter.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ template "prometheus-node-exporter.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc -w {{ template "prometheus-node-exporter.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ template "prometheus-node-exporter.namespace" . }} {{ template "prometheus-node-exporter.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ template "prometheus-node-exporter.namespace" . }} -l "app.kubernetes.io/name={{ template "prometheus-node-exporter.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + echo "Visit http://127.0.0.1:{{ .Values.service.port }} to use your application" + kubectl port-forward --namespace {{ template "prometheus-node-exporter.namespace" . }} $POD_NAME {{ .Values.service.port }} +{{- end }} + +{{- if .Values.kubeRBACProxy.enabled}} + +kube-rbac-proxy endpoint protections is enabled: +- Metrics endpoints is now HTTPS +- Ensure that the client authenticates the requests (e.g. via service account) with the following role permissions: +``` +rules: + - apiGroups: [ "" ] + resources: ["services/{{ template "prometheus-node-exporter.fullname" . }}"] + verbs: + - get +``` +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/_helpers.tpl new file mode 100644 index 0000000000..72a6db45a1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/_helpers.tpl @@ -0,0 +1,236 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "prometheus-node-exporter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "prometheus-node-exporter.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "prometheus-node-exporter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "prometheus-node-exporter.labels" -}} +helm.sh/chart: {{ include "prometheus-node-exporter.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ include "prometheus-node-exporter.name" . }} +{{ include "prometheus-node-exporter.selectorLabels" . }} +{{- with .Chart.AppVersion }} +app.kubernetes.io/version: {{ . | quote }} +{{- end }} +{{- with .Values.podLabels }} +{{ toYaml . }} +{{- end }} +{{- if .Values.releaseLabel }} +release: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "prometheus-node-exporter.selectorLabels" -}} +app.kubernetes.io/name: {{ include "prometheus-node-exporter.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + +{{/* +Create the name of the service account to use +*/}} +{{- define "prometheus-node-exporter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "prometheus-node-exporter.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +The image to use +*/}} +{{- define "prometheus-node-exporter.image" -}} +{{- $temp_registry := (include "system_default_registry" .) }} +{{- if .Values.image.sha }} +{{- fail "image.sha forbidden. Use image.digest instead" }} +{{- else if .Values.image.digest }} +{{- if $temp_registry }} +{{- printf "%s%s:%s@%s" $temp_registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.digest }} +{{- else if .Values.global.imageRegistry }} +{{- printf "%s/%s:%s@%s" .Values.global.imageRegistry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.digest }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.digest }} +{{- end }} +{{- else }} +{{- if $temp_registry }} +{{- printf "%s%s:%s" $temp_registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- else if .Values.global.imageRegistry }} +{{- printf "%s/%s:%s" .Values.global.imageRegistry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "prometheus-node-exporter.namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} + +{{/* +Create the namespace name of the service monitor +*/}} +{{- define "prometheus-node-exporter.monitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.monitor.namespace }} +{{- .Values.prometheus.monitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} + +{{/* +Formats imagePullSecrets. Input is (dict "Values" .Values "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "prometheus-node-exporter.imagePullSecrets" -}} +{{- range (concat .Values.global.imagePullSecrets .imagePullSecrets) }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +Create the namespace name of the pod monitor +*/}} +{{- define "prometheus-node-exporter.podmonitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.podMonitor.namespace }} +{{- .Values.prometheus.podMonitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for podmonitor */}} +{{- define "podmonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} + +{{/* Sets sidecar volumeMounts */}} +{{- define "prometheus-node-exporter.sidecarVolumeMounts" -}} +{{- range $_, $mount := $.Values.sidecarVolumeMount }} +- name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} +{{- end }} +{{- range $_, $mount := $.Values.sidecarHostVolumeMounts }} +- name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} +{{- if $mount.mountPropagation }} + mountPropagation: {{ $mount.mountPropagation }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/clusterrole.yaml new file mode 100644 index 0000000000..c256dba73d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/clusterrole.yaml @@ -0,0 +1,19 @@ +{{- if and (eq .Values.rbac.create true) (eq .Values.kubeRBACProxy.enabled true) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +rules: + {{- if $.Values.kubeRBACProxy.enabled }} + - apiGroups: [ "authentication.k8s.io" ] + resources: + - tokenreviews + verbs: [ "create" ] + - apiGroups: [ "authorization.k8s.io" ] + resources: + - subjectaccessreviews + verbs: [ "create" ] + {{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..653305ad9e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and (eq .Values.rbac.create true) (eq .Values.kubeRBACProxy.enabled true) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + name: {{ template "prometheus-node-exporter.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole +{{- if .Values.rbac.useExistingRole }} + name: {{ .Values.rbac.useExistingRole }} +{{- else }} + name: {{ template "prometheus-node-exporter.fullname" . }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "prometheus-node-exporter.serviceAccountName" . }} + namespace: {{ template "prometheus-node-exporter.namespace" . }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/daemonset.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/daemonset.yaml new file mode 100644 index 0000000000..48d274f1b1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/daemonset.yaml @@ -0,0 +1,309 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.daemonsetAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- with .Values.updateStrategy }} + updateStrategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 8 }} + spec: + automountServiceAccountToken: {{ ternary true false (or .Values.serviceAccount.automountServiceAccountToken .Values.kubeRBACProxy.enabled) }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.extraInitContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "prometheus-node-exporter.serviceAccountName" . }} + {{- with .Values.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ . }} + {{- end }} + containers: + {{- $servicePort := ternary .Values.kubeRBACProxy.port .Values.service.port .Values.kubeRBACProxy.enabled }} + - name: node-exporter + image: {{ include "prometheus-node-exporter.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --path.procfs=/host/proc + - --path.sysfs=/host/sys + {{- if .Values.hostRootFsMount.enabled }} + - --path.rootfs=/host/root + {{- if semverCompare ">=1.4.0-0" (coalesce .Values.version .Values.image.tag .Chart.AppVersion) }} + - --path.udev.data=/host/root/run/udev/data + {{- end }} + {{- end }} + - --web.listen-address=[$(HOST_IP)]:{{ $servicePort }} + {{- with .Values.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + - name: HOST_IP + {{- if .Values.kubeRBACProxy.enabled }} + value: 127.0.0.1 + {{- else if .Values.service.listenOnAllInterfaces }} + value: 0.0.0.0 + {{- else }} + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + {{- end }} + {{- range $key, $value := .Values.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + {{- if eq .Values.kubeRBACProxy.enabled false }} + ports: + - name: {{ .Values.service.portName }} + containerPort: {{ .Values.service.port }} + protocol: TCP + {{- end }} + livenessProbe: + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + httpGet: + {{- if .Values.kubeRBACProxy.enabled }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.livenessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ $servicePort }} + scheme: {{ upper .Values.livenessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + readinessProbe: + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + httpGet: + {{- if .Values.kubeRBACProxy.enabled }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.readinessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ $servicePort }} + scheme: {{ upper .Values.readinessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.terminationMessageParams.enabled }} + {{- with .Values.terminationMessageParams }} + terminationMessagePath: {{ .terminationMessagePath }} + terminationMessagePolicy: {{ .terminationMessagePolicy }} + {{- end }} + {{- end }} + volumeMounts: + - name: proc + mountPath: /host/proc + {{- with .Values.hostProcFsMount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + readOnly: true + - name: sys + mountPath: /host/sys + {{- with .Values.hostSysFsMount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + readOnly: true + {{- if .Values.hostRootFsMount.enabled }} + - name: root + mountPath: /host/root + {{- with .Values.hostRootFsMount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + readOnly: true + {{- end }} + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- with $mount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: true + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + {{- range .Values.sidecars }} + {{- $overwrites := dict "volumeMounts" (concat (include "prometheus-node-exporter.sidecarVolumeMounts" $ | fromYamlArray) (.volumeMounts | default list) | default list) }} + {{- $defaults := dict "image" (include "prometheus-node-exporter.image" $) "securityContext" $.Values.containerSecurityContext "imagePullPolicy" $.Values.image.pullPolicy }} + - {{- toYaml (merge $overwrites . $defaults) | nindent 10 }} + {{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - name: kube-rbac-proxy + args: + {{- if .Values.kubeRBACProxy.extraArgs }} + {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 12 }} + {{- end }} + - --secure-listen-address=:{{ .Values.service.port}} + - --upstream=http://127.0.0.1:{{ $servicePort }}/ + - --proxy-endpoints-port=8888 + - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml + volumeMounts: + - name: kube-rbac-proxy-config + mountPath: /etc/kube-rbac-proxy-config + imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }} + {{- $base_registry := (include "monitoring_registry" .) }} + {{- if .Values.kubeRBACProxy.image.sha }} + image: "{{ $base_registry | default .Values.kubeRBACProxy.image.registry}}/{{ .Values.kubeRBACProxy.image.repository }}:{{ .Values.kubeRBACProxy.image.tag }}@sha256:{{ .Values.kubeRBACProxy.image.sha }}" + {{- else }} + image: "{{ $base_registry | default .Values.kubeRBACProxy.image.registry}}/{{ .Values.kubeRBACProxy.image.repository }}:{{ .Values.kubeRBACProxy.image.tag }}" + {{- end }} + ports: + - containerPort: {{ .Values.service.port}} + name: {{ .Values.kubeRBACProxy.portName }} + {{- if .Values.kubeRBACProxy.enableHostPort }} + hostPort: {{ .Values.service.port }} + {{- end }} + - containerPort: 8888 + name: "http-healthz" + readinessProbe: + httpGet: + scheme: HTTPS + port: 8888 + path: healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + {{- if .Values.kubeRBACProxy.resources }} + resources: + {{- toYaml .Values.kubeRBACProxy.resources | nindent 12 }} + {{- end }} + {{- if .Values.terminationMessageParams.enabled }} + {{- with .Values.terminationMessageParams }} + terminationMessagePath: {{ .terminationMessagePath }} + terminationMessagePolicy: {{ .terminationMessagePolicy }} + {{- end }} + {{- end }} + {{- with .Values.kubeRBACProxy.env }} + env: + {{- range $key, $value := $.Values.kubeRBACProxy.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + {{- end }} + {{- if .Values.kubeRBACProxy.containerSecurityContext }} + securityContext: + {{ toYaml .Values.kubeRBACProxy.containerSecurityContext | nindent 12 }} + {{- end }} + {{- end }} + {{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "prometheus-node-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.imagePullSecrets) | indent 8 }} + {{- end }} + hostNetwork: {{ .Values.hostNetwork }} + hostPID: {{ .Values.hostPID }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.dnsConfig }} + dnsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.restartPolicy }} + restartPolicy: {{ . }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: proc + hostPath: + path: /proc + - name: sys + hostPath: + path: /sys + {{- if .Values.hostRootFsMount.enabled }} + - name: root + hostPath: + path: / + {{- end }} + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- with $mount.type }} + type: {{ . }} + {{- end }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + emptyDir: + medium: Memory + {{- end }} + {{- range $_, $mount := .Values.sidecarHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + configMap: + name: {{ $mount.name }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ $mount.name }} + secret: + secretName: {{ $mount.name }} + {{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - name: kube-rbac-proxy-config + configMap: + name: {{ template "prometheus-node-exporter.fullname" . }}-rbac-config + {{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/endpoints.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/endpoints.yaml new file mode 100644 index 0000000000..56b695203a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/endpoints.yaml @@ -0,0 +1,18 @@ +{{- if .Values.endpoints }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +subsets: + - addresses: + {{- range .Values.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.service.portName }} + port: {{ .Values.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/extra-manifests.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/extra-manifests.yaml new file mode 100644 index 0000000000..2b21b71062 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraManifests }} +--- +{{ tpl . $ }} +{{ end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/networkpolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/networkpolicy.yaml new file mode 100644 index 0000000000..825722729d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/networkpolicy.yaml @@ -0,0 +1,23 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ingress: + - ports: + - port: {{ .Values.service.port }} + policyTypes: + - Egress + - Ingress + podSelector: + matchLabels: + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/podmonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/podmonitor.yaml new file mode 100644 index 0000000000..f88da6a34e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/podmonitor.yaml @@ -0,0 +1,91 @@ +{{- if .Values.prometheus.podMonitor.enabled }} +apiVersion: {{ .Values.prometheus.podMonitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: PodMonitor +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.podmonitor-namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.podMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.podMonitor.jobLabel }} + {{- include "podmonitor.scrapeLimits" .Values.prometheus.podMonitor | nindent 2 }} + selector: + matchLabels: + {{- with .Values.prometheus.podMonitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ include "prometheus-node-exporter.namespace" . }} + {{- with .Values.prometheus.podMonitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + podMetricsEndpoints: + - port: {{ .Values.service.portName }} + {{- with .Values.prometheus.podMonitor.scheme }} + scheme: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.path }} + path: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.authorization }} + authorization: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.oauth2 }} + oauth2: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorTimestamps }} + honorTimestamps: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorLabels }} + honorLabels: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + enableHttp2: {{ default false .Values.prometheus.podMonitor.enableHttp2 }} + filterRunning: {{ default true .Values.prometheus.podMonitor.filterRunning }} + followRedirects: {{ default false .Values.prometheus.podMonitor.followRedirects }} + {{- with .Values.prometheus.podMonitor.params }} + params: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml new file mode 100644 index 0000000000..ee5bbba4a5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml @@ -0,0 +1,14 @@ +{{- if and (or .Values.global.cattle.psp.enable (and .Values.rbac.create .Values.rbac.pspEnabled)) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: psp-{{ include "prometheus-node-exporter.fullname" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +rules: +- apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ include "prometheus-node-exporter.fullname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..160f2bbf7a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if and (or .Values.global.cattle.psp.enable (and .Values.rbac.create .Values.rbac.pspEnabled)) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: psp-{{ include "prometheus-node-exporter.fullname" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp-{{ include "prometheus-node-exporter.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/psp.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/psp.yaml new file mode 100644 index 0000000000..f3b52e1120 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/psp.yaml @@ -0,0 +1,49 @@ +{{- if and (or .Values.global.cattle.psp.enable (and .Values.rbac.create .Values.rbac.pspEnabled)) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.rbac.pspAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + - 'hostPath' + hostNetwork: true + hostIPC: false + hostPID: true + hostPorts: + - min: 0 + max: 65535 + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/rbac-configmap.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/rbac-configmap.yaml new file mode 100644 index 0000000000..814e110337 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/rbac-configmap.yaml @@ -0,0 +1,16 @@ +{{- if .Values.kubeRBACProxy.enabled}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "prometheus-node-exporter.fullname" . }}-rbac-config + namespace: {{ include "prometheus-node-exporter.namespace" . }} +data: + config-file.yaml: |+ + authorization: + resourceAttributes: + namespace: {{ template "prometheus-node-exporter.namespace" . }} + apiVersion: v1 + resource: services + subresource: {{ template "prometheus-node-exporter.fullname" . }} + name: {{ template "prometheus-node-exporter.fullname" . }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/service.yaml new file mode 100644 index 0000000000..a065e46e39 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/service.yaml @@ -0,0 +1,29 @@ +{{- if .Values.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: +{{- if .Values.service.ipDualStack.enabled }} + ipFamilies: {{ toYaml .Values.service.ipDualStack.ipFamilies | nindent 4 }} + ipFamilyPolicy: {{ .Values.service.ipDualStack.ipFamilyPolicy }} +{{- end }} + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + {{- if ( and (eq .Values.service.type "NodePort" ) (not (empty .Values.service.nodePort)) ) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: {{ .Values.service.targetPort }} + protocol: TCP + name: {{ .Values.service.portName }} + selector: + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/serviceaccount.yaml new file mode 100644 index 0000000000..5c3348c09b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.rbac.create .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "prometheus-node-exporter.serviceAccountName" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if or .Values.serviceAccount.imagePullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "prometheus-node-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.serviceAccount.imagePullSecrets) | indent 2 }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/servicemonitor.yaml new file mode 100644 index 0000000000..6d6e440473 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/servicemonitor.yaml @@ -0,0 +1,71 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: {{ .Values.prometheus.monitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: ServiceMonitor +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.monitor-namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.monitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.monitor | nindent 2 }} + {{- with .Values.prometheus.monitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- with .Values.prometheus.monitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} + {{- end }} + {{- with .Values.prometheus.monitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + endpoints: + - port: {{ .Values.service.portName }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- with .Values.prometheus.monitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + metricRelabelings: + {{- with .Values.prometheus.monitor.metricRelabelings }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml new file mode 100644 index 0000000000..2c2705f872 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml @@ -0,0 +1,40 @@ +{{- if and (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1") (.Values.verticalPodAutoscaler.enabled) }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +spec: + {{- with .Values.verticalPodAutoscaler.recommenders }} + recommenders: + {{- toYaml . | nindent 4 }} + {{- end }} + resourcePolicy: + containerPolicies: + - containerName: node-exporter + {{- with .Values.verticalPodAutoscaler.controlledResources }} + controlledResources: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.verticalPodAutoscaler.controlledValues }} + controlledValues: {{ . }} + {{- end }} + {{- with .Values.verticalPodAutoscaler.maxAllowed }} + maxAllowed: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.verticalPodAutoscaler.minAllowed }} + minAllowed: + {{- toYaml . | nindent 8 }} + {{- end }} + targetRef: + apiVersion: apps/v1 + kind: DaemonSet + name: {{ include "prometheus-node-exporter.fullname" . }} + {{- with .Values.verticalPodAutoscaler.updatePolicy }} + updatePolicy: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/values.yaml new file mode 100644 index 0000000000..b9f2f7ab87 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/prometheus-node-exporter/values.yaml @@ -0,0 +1,530 @@ +# Default values for prometheus-node-exporter. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + registry: docker.io + repository: rancher/mirrored-prometheus-node-exporter + # Overrides the image tag whose default is {{ printf "v%s" .Chart.AppVersion }} + tag: v1.7.0 + pullPolicy: IfNotPresent + digest: "" + +imagePullSecrets: [] +# - name: "image-pull-secret" +nameOverride: "" +fullnameOverride: "" + +# Number of old history to retain to allow rollback +# Default Kubernetes value is set to 10 +revisionHistoryLimit: 10 + +global: + cattle: + psp: + enable: true + systemDefaultRegistry: "" + + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + # + # Allow parent charts to override registry hostname + imageRegistry: "docker.io" + +# Configure kube-rbac-proxy. When enabled, creates a kube-rbac-proxy to protect the node-exporter http endpoint. +# The requests are served through the same service but requests are HTTPS. +kubeRBACProxy: + enabled: false + ## Set environment variables as name/value pairs + env: {} + # VARIABLE: value + image: + registry: docker.io + repository: rancher/mirrored-kube-rbac-proxy + tag: v0.15.0 + sha: "" + pullPolicy: IfNotPresent + + # List of additional cli arguments to configure kube-rbac-proxy + # for example: --tls-cipher-suites, --log-file, etc. + # all the possible args can be found here: https://github.com/brancz/kube-rbac-proxy#usage + extraArgs: [] + + ## Specify security settings for a Container + ## Allows overrides and additional options compared to (Pod) securityContext + ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + containerSecurityContext: {} + + # Specify the port used for the Node exporter container (upstream port) + port: 8100 + # Specify the name of the container port + portName: http + # Configure a hostPort. If true, hostPort will be enabled in the container and set to service.port. + enableHostPort: false + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 64Mi + # requests: + # cpu: 10m + # memory: 32Mi + +service: + enabled: true + type: ClusterIP + port: 9796 + targetPort: 9796 + nodePort: + portName: metrics + listenOnAllInterfaces: true + annotations: + prometheus.io/scrape: "true" + ipDualStack: + enabled: false + ipFamilies: ["IPv6", "IPv4"] + ipFamilyPolicy: "PreferDualStack" + +# Set a NetworkPolicy with: +# ingress only on service.port +# no egress permitted +networkPolicy: + enabled: false + +# Additional environment variables that will be passed to the daemonset +env: {} +## env: +## VARIABLE: value + +prometheus: + monitor: + enabled: false + additionalLabels: {} + namespace: "" + + jobLabel: "" + + # List of pod labels to add to node exporter metrics + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor + podTargetLabels: [] + + scheme: http + basicAuth: {} + bearerTokenFile: + tlsConfig: {} + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## Override serviceMonitor selector + ## + selectorOverride: {} + + ## Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + ## + attachMetadata: + node: false + + relabelings: [] + metricRelabelings: [] + interval: "" + scrapeTimeout: 10s + ## prometheus.monitor.apiVersion ApiVersion for the serviceMonitor Resource(defaults to "monitoring.coreos.com/v1") + apiVersion: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + # PodMonitor defines monitoring for a set of pods. + # ref. https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.PodMonitor + # Using a PodMonitor may be preferred in some environments where there is very large number + # of Node Exporter endpoints (1000+) behind a single service. + # The PodMonitor is disabled by default. When switching from ServiceMonitor to PodMonitor, + # the time series resulting from the configuration through PodMonitor may have different labels. + # For instance, there will not be the service label any longer which might + # affect PromQL queries selecting that label. + podMonitor: + enabled: false + # Namespace in which to deploy the pod monitor. Defaults to the release namespace. + namespace: "" + # Additional labels, e.g. setting a label for pod monitor selector as set in prometheus + additionalLabels: {} + # release: kube-prometheus-stack + # PodTargetLabels transfers labels of the Kubernetes Pod onto the target. + podTargetLabels: [] + # apiVersion defaults to monitoring.coreos.com/v1. + apiVersion: "" + # Override pod selector to select pod objects. + selectorOverride: {} + # Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + attachMetadata: + node: false + # The label to use to retrieve the job name from. Defaults to label app.kubernetes.io/name. + jobLabel: "" + + # Scheme/protocol to use for scraping. + scheme: "http" + # Path to scrape metrics at. + path: "/metrics" + + # BasicAuth allow an endpoint to authenticate over basic authentication. + # More info: https://prometheus.io/docs/operating/configuration/#endpoint + basicAuth: {} + # Secret to mount to read bearer token for scraping targets. + # The secret needs to be in the same namespace as the pod monitor and accessible by the Prometheus Operator. + # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secretkeyselector-v1-core + bearerTokenSecret: {} + # TLS configuration to use when scraping the endpoint. + tlsConfig: {} + # Authorization section for this endpoint. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.SafeAuthorization + authorization: {} + # OAuth2 for the URL. Only valid in Prometheus versions 2.27.0 and newer. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.OAuth2 + oauth2: {} + + # ProxyURL eg http://proxyserver:2195. Directs scrapes through proxy to this endpoint. + proxyUrl: "" + # Interval at which endpoints should be scraped. If not specified Prometheus’ global scrape interval is used. + interval: "" + # Timeout after which the scrape is ended. If not specified, the Prometheus global scrape interval is used. + scrapeTimeout: "" + # HonorTimestamps controls whether Prometheus respects the timestamps present in scraped data. + honorTimestamps: true + # HonorLabels chooses the metric’s labels on collisions with target labels. + honorLabels: true + # Whether to enable HTTP2. Default false. + enableHttp2: "" + # Drop pods that are not running. (Failed, Succeeded). + # Enabled by default. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase + filterRunning: "" + # FollowRedirects configures whether scrape requests follow HTTP 3xx redirects. Default false. + followRedirects: "" + # Optional HTTP URL parameters + params: {} + + # RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds + # relabelings for a few standard Kubernetes fields. The original scrape job’s name + # is available via the __tmp_prometheus_job_name label. + # More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + relabelings: [] + # MetricRelabelConfigs to apply to samples before ingestion. + metricRelabelings: [] + + # SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + sampleLimit: 0 + # TargetLimit defines a limit on the number of scraped targets that will be accepted. + targetLimit: 0 + # Per-scrape limit on number of labels that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelLimit: 0 + # Per-scrape limit on length of labels name that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelNameLengthLimit: 0 + # Per-scrape limit on length of labels value that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelValueLengthLimit: 0 + +## Customize the updateStrategy if set +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m + # memory: 30Mi + +# Specify the container restart policy passed to the Node Export container +# Possible Values: Always (default)|OnFailure|Never +restartPolicy: null + +serviceAccount: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + annotations: {} + imagePullSecrets: [] + automountServiceAccountToken: false + +securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + +containerSecurityContext: + readOnlyRootFilesystem: true + # capabilities: + # add: + # - SYS_TIME + +rbac: + ## If true, create & use RBAC resources + ## + create: true + pspAnnotations: {} + +# for deployments that have node_exporter deployed outside of the cluster, list +# their addresses here +endpoints: [] + +# Expose the service to the host network +hostNetwork: true + +# Share the host process ID namespace +hostPID: true + +# Mount the node's root file system (/) at /host/root in the container +hostRootFsMount: + enabled: true + # Defines how new mounts in existing mounts on the node or in the container + # are propagated to the container or node, respectively. Possible values are + # None, HostToContainer, and Bidirectional. If this field is omitted, then + # None is used. More information on: + # https://kubernetes.io/docs/concepts/storage/volumes/#mount-propagation + mountPropagation: HostToContainer + +# Mount the node's proc file system (/proc) at /host/proc in the container +hostProcFsMount: + # Possible values are None, HostToContainer, and Bidirectional + mountPropagation: "" + +# Mount the node's sys file system (/sys) at /host/sys in the container +hostSysFsMount: + # Possible values are None, HostToContainer, and Bidirectional + mountPropagation: "" + +## Assign a group of affinity scheduling rules +## +affinity: {} +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchFields: +# - key: metadata.name +# operator: In +# values: +# - target-host-name + +# Annotations to be added to node exporter pods +podAnnotations: + # Fix for very slow GKE cluster upgrades + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + +# Extra labels to be added to node exporter pods +podLabels: {} + +# Annotations to be added to node exporter daemonset +daemonsetAnnotations: {} + +## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box +releaseLabel: false + +# Custom DNS configuration to be added to prometheus-node-exporter pods +dnsConfig: {} +# nameservers: +# - 1.2.3.4 +# searches: +# - ns1.svc.cluster-domain.example +# - my.dns.search.suffix +# options: +# - name: ndots +# value: "2" +# - name: edns0 + +## Assign a nodeSelector if operating a hybrid cluster +## +nodeSelector: + kubernetes.io/os: linux + # kubernetes.io/arch: amd64 + +# Specify grace period for graceful termination of pods. Defaults to 30 if null or not specified +terminationGracePeriodSeconds: null + +tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + +# Enable or disable container termination message settings +# https://kubernetes.io/docs/tasks/debug/debug-application/determine-reason-pod-failure/ +terminationMessageParams: + enabled: false + # If enabled, specify the path for termination messages + terminationMessagePath: /dev/termination-log + # If enabled, specify the policy for termination messages + terminationMessagePolicy: File + + +## Assign a PriorityClassName to pods if set +# priorityClassName: "" + +## Additional container arguments +## +extraArgs: [] +# - --collector.diskstats.ignored-devices=^(ram|loop|fd|(h|s|v)d[a-z]|nvme\\d+n\\d+p)\\d+$ +# - --collector.textfile.directory=/run/prometheus + +## Additional mounts from the host to node-exporter container +## +extraHostVolumeMounts: [] +# - name: +# hostPath: +# https://kubernetes.io/docs/concepts/storage/volumes/#hostpath-volume-types +# type: "" (Default)|DirectoryOrCreate|Directory|FileOrCreate|File|Socket|CharDevice|BlockDevice +# mountPath: +# readOnly: true|false +# mountPropagation: None|HostToContainer|Bidirectional + +## Additional configmaps to be mounted. +## +configmaps: [] +# - name: +# mountPath: +secrets: [] +# - name: +# mountPath: +## Override the deployment namespace +## +namespaceOverride: "" + +## Additional containers for export metrics to text file; fields image,imagePullPolicy,securityContext take default value from main container +## +sidecars: [] +# - name: nvidia-dcgm-exporter +# image: nvidia/dcgm-exporter:1.4.3 +# volumeMounts: +# - name: tmp +# mountPath: /tmp + +## Volume for sidecar containers +## +sidecarVolumeMount: [] +# - name: collector-textfiles +# mountPath: /run/prometheus +# readOnly: false + +## Additional mounts from the host to sidecar containers +## +sidecarHostVolumeMounts: [] +# - name: +# hostPath: +# mountPath: +# readOnly: true|false +# mountPropagation: None|HostToContainer|Bidirectional + +## Additional InitContainers to initialize the pod +## +extraInitContainers: [] + +## Liveness probe +## +livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + +## Readiness probe +## +readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + +# Enable vertical pod autoscaler support for prometheus-node-exporter +verticalPodAutoscaler: + enabled: false + + # Recommender responsible for generating recommendation for the object. + # List should be empty (then the default recommender will generate the recommendation) + # or contain exactly one recommender. + # recommenders: + # - name: custom-recommender-performance + + # List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory + controlledResources: [] + # Specifies which resource values should be controlled: RequestsOnly or RequestsAndLimits. + # controlledValues: RequestsAndLimits + + # Define the max allowed resources for the pod + maxAllowed: {} + # cpu: 200m + # memory: 100Mi + # Define the min allowed resources for the pod + minAllowed: {} + # cpu: 200m + # memory: 100Mi + + # updatePolicy: + # Specifies minimal number of replicas which need to be alive for VPA Updater to attempt pod eviction + # minReplicas: 1 + # Specifies whether recommended updates are applied when a Pod is started and whether recommended updates + # are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto". + # updateMode: Auto + +# Extra manifests to deploy as an array +extraManifests: [] + # - | + # apiVersion: v1 + # kind: ConfigMap + # metadata: + # name: prometheus-extra + # data: + # extra-data: "value" + +# Override version of app, required if image.tag is defined and does not follow semver +version: "" diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/Chart.yaml new file mode 100644 index 0000000000..dbf4d0b815 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2ControllerManager +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2ControllerManager/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/Chart.yaml new file mode 100644 index 0000000000..87495a6a6b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2Etcd +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Etcd/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/Chart.yaml new file mode 100644 index 0000000000..4bce6d41ea --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2IngressNginx +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2IngressNginx/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/Chart.yaml new file mode 100644 index 0000000000..0a316e05c7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2Proxy +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Proxy/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/Chart.yaml new file mode 100644 index 0000000000..fa6aa5ac66 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2Scheduler +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rke2Scheduler/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/Chart.yaml new file mode 100644 index 0000000000..df00a46b66 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeControllerManager +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeControllerManager/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/Chart.yaml new file mode 100644 index 0000000000..96b33f0bc8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeEtcd +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeEtcd/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/Chart.yaml new file mode 100644 index 0000000000..bc49bcefd2 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeIngressNginx +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeIngressNginx/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/Chart.yaml new file mode 100644 index 0000000000..18eac324d4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeProxy +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeProxy/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/.helmignore new file mode 100644 index 0000000000..0e8a0eb36f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/Chart.yaml new file mode 100644 index 0000000000..8c53b63bcc --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeScheduler +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/README.md new file mode 100644 index 0000000000..345002f48a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/_helpers.tpl new file mode 100644 index 0000000000..1ba5093944 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml new file mode 100644 index 0000000000..a8e27c3735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-clients.yaml new file mode 100644 index 0000000000..e8fcfb3883 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 0000000000..eefe609058 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy.yaml new file mode 100644 index 0000000000..723bbd6c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml new file mode 100644 index 0000000000..67eb2216b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..16abc2fa83 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/values.yaml new file mode 100644 index 0000000000..1e076041b3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/rkeScheduler/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.3-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.3-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/.helmignore b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/Chart.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/Chart.yaml new file mode 100644 index 0000000000..784bb0ec7e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: v2 +appVersion: 0.25.1 +description: A Helm chart for prometheus windows-exporter +home: https://github.com/prometheus-community/windows_exporter/ +keywords: +- windows-exporter +- windows +- prometheus +- exporter +maintainers: +- email: github@jkroepke.de + name: jkroepke +name: windowsExporter +sources: +- https://github.com/prometheus-community/windows_exporter/ +type: application +version: 0.3.1 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/README.md b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/README.md new file mode 100644 index 0000000000..1da1c64e12 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/README.md @@ -0,0 +1,42 @@ +# Prometheus `Windows Exporter` + +Prometheus exporter for hardware and OS metrics exposed by Windows kernels, written in Go with pluggable metric collectors. + +This chart bootstraps a prometheus [`Windows Exporter`](http://github.com/prometheus-community/windows_exporter) daemonset on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Get Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Chart + +```console +helm install [RELEASE_NAME] prometheus-community/prometheus-windows-exporter +``` + +_See [configuration](#configuring) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Configuring + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values prometheus-community/prometheus-windows-exporter +``` diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/scripts/configure-firewall.ps1 b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/scripts/configure-firewall.ps1 new file mode 100644 index 0000000000..9cbed7112d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/scripts/configure-firewall.ps1 @@ -0,0 +1,31 @@ +$ErrorActionPreference = 'Continue' + +function CheckFirewallRuleError { + # We hit an error. This can happen for a number of reasons, including if the rule already exists + if ($error[0]) { + if (($error[0].Exception.NativeErrorCode) -and ($error[0].Exception.NativeErrorCode.ToString() -eq "AlreadyExists")) { + # Previous versions of monitoring may have already created this Firewall Rule + # Because of this, if the rule alreadys exists there is no need to delete and recreate it. + Write-Host "Detected Existing Firewall Rule, Nothing To Do" + } else { + Write-Host "Error Encountered Setting Up Required Firewall Rule" + $error[0].Exception + exit 1 + } + } +} + +Write-Host "Attempting To Configure Firewall Rules For Ports 9796, 10250" + +# This is the exact same firewall rule that has historically been created by rancher-wins +# https://github.com/rancher/wins/blob/91f670c47f19c6d9fe97d8f66a695d3081ad994f/pkg/apis/process_service_mgmt.go#L149 +New-NetFirewallRule -DisplayName rancher-wins-windows-exporter-TCP-9796 -Name rancher-wins-windows-exporter-TCP-9796 -Action Allow -Protocol TCP -LocalPort 9796 -Enabled True -PolicyStore ActiveStore +CheckFirewallRuleError +Write-Host "Windows Node Exporter Firewall Rule Successfully Created" + +# This rule is required in order to have the Rancher UI display node metrics in the 'Nodes' tab of the cluster explorer +New-NetFirewallRule -DisplayName rancher-wins-windows-exporter-TCP-10250 -Name rancher-wins-windows-exporter-TCP-10250 -Action Allow -Protocol TCP -LocalPort 10250 -Enabled True -PolicyStore ActiveStore +CheckFirewallRuleError +Write-Host "Windows Prometheus Metrics Firewall Rule Successfully Created" + +Write-Host "All Firewall Rules Successfully Configured" diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/_helpers.tpl new file mode 100644 index 0000000000..c9a5d6db8c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/_helpers.tpl @@ -0,0 +1,216 @@ +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +The components in this chart create additional resources that expand the longest created name strings. +The longest name that gets created adds and extra 37 characters, so truncation should be 63-35=26. +*/}} +{{- define "prometheus-windows-exporter.fullname" -}} +{{ printf "%s-windows-exporter" .Release.Name }} +{{- end -}} + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{- define "windowsExporter.renamedMetricsRelabeling" -}} +{{- range $original, $new := (include "windowsExporter.renamedMetrics" . | fromJson) -}} +- sourceLabels: [__name__] + regex: {{ $original }} + replacement: '{{ $new }}' + targetLabel: __name__ +{{ end -}} +{{- end -}} + +{{- define "windowsExporter.labels" -}} +k8s-app: {{ template "prometheus-windows-exporter.fullname" . }} +release: {{ .Release.Name }} +component: "windows-exporter" +provider: kubernetes +{{- end -}} + +{{- define "windowsExporter.renamedMetrics" -}} +{{- $renamed := dict -}} +{{/* v0.15.0 */}} +{{- $_ := set $renamed "windows_mssql_transactions_active_total" "windows_mssql_transactions_active" -}} +{{/* v0.16.0 */}} +{{- $_ := set $renamed "windows_adfs_ad_login_connection_failures" "windows_adfs_ad_login_connection_failures_total" -}} +{{- $_ := set $renamed "windows_adfs_certificate_authentications" "windows_adfs_certificate_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_device_authentications" "windows_adfs_device_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_extranet_account_lockouts" "windows_adfs_extranet_account_lockouts_total" -}} +{{- $_ := set $renamed "windows_adfs_federated_authentications" "windows_adfs_federated_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_passport_authentications" "windows_adfs_passport_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_password_change_failed" "windows_adfs_password_change_failed_total" -}} +{{- $_ := set $renamed "windows_adfs_password_change_succeeded" "windows_adfs_password_change_succeeded_total" -}} +{{- $_ := set $renamed "windows_adfs_token_requests" "windows_adfs_token_requests_total" -}} +{{- $_ := set $renamed "windows_adfs_windows_integrated_authentications" "windows_adfs_windows_integrated_authentications_total" -}} +{{- $_ := set $renamed "windows_net_packets_outbound_errors" "windows_net_packets_outbound_errors_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_discarded" "windows_net_packets_received_discarded_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_errors" "windows_net_packets_received_errors_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_total" "windows_net_packets_received_total_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_unknown" "windows_net_packets_received_unknown_total" -}} +{{- $_ := set $renamed "windows_dns_memory_used_bytes_total" "windows_dns_memory_used_bytes" -}} +{{- $renamed | toJson -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "prometheus-windows-exporter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "prometheus-windows-exporter.labels" -}} +helm.sh/chart: {{ include "prometheus-windows-exporter.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ include "prometheus-windows-exporter.name" . }} +{{ include "prometheus-windows-exporter.selectorLabels" . }} +{{- with .Chart.AppVersion }} +app.kubernetes.io/version: {{ . | quote }} +{{- end }} +{{- with .Values.podLabels }} +{{ toYaml . }} +{{- end }} +{{- if .Values.releaseLabel }} +release: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "prometheus-windows-exporter.selectorLabels" -}} +app.kubernetes.io/name: {{ include "prometheus-windows-exporter.fullname" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + +{{/* +Create the name of the service account to use +*/}} +{{- define "prometheus-windows-exporter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "prometheus-windows-exporter.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +The image to use +*/}} +{{- define "prometheus-windows-exporter.image" -}} +{{- if .Values.image.sha }} +{{- fail "image.sha forbidden. Use image.digest instead" }} +{{- else if .Values.image.digest }} +{{- if .Values.global.cattle.systemDefaultRegistry }} +{{- printf "%s/%s:%s@%s" .Values.global.cattle.systemDefaultRegistry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) .Values.image.digest }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.image.registry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) .Values.image.digest }} +{{- end }} +{{- else }} +{{- if .Values.global.cattle.systemDefaultRegistry }} +{{- printf "%s/%s:%s" .Values.global.cattle.systemDefaultRegistry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.image.registry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "prometheus-windows-exporter.namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} + +{{/* +Create the namespace name of the service monitor +*/}} +{{- define "prometheus-windows-exporter.monitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.monitor.namespace }} +{{- .Values.prometheus.monitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} + +{{/* +Formats imagePullSecrets. Input is (dict "Values" .Values "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "prometheus-windows-exporter.imagePullSecrets" -}} +{{- range (concat .Values.global.imagePullSecrets .imagePullSecrets) }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +Create the namespace name of the pod monitor +*/}} +{{- define "prometheus-windows-exporter.podmonitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.podMonitor.namespace }} +{{- .Values.prometheus.podMonitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for podmonitor */}} +{{- define "podmonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/config.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/config.yaml new file mode 100644 index 0000000000..25f1fa69c2 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/config.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + config.yml: | + {{- .Values.config | nindent 4 }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/daemonset.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/daemonset.yaml new file mode 100644 index 0000000000..be7feb3ed1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/daemonset.yaml @@ -0,0 +1,200 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.daemonsetAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "windowsExporter.labels" . | nindent 6 }} + {{- with .Values.updateStrategy }} + updateStrategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "windowsExporter.labels" . | nindent 8 }} + spec: + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + initContainers: + - name: configure-firewall + image: {{ include "prometheus-windows-exporter.image" . }} + command: + - C:\WINDOWS\System32\WindowsPowerShell\v1.0\powershell.exe + args: ["-f", "scripts/configure-firewall.ps1"] + volumeMounts: + - mountPath: /scripts + name: exporter-scripts + {{- with .Values.extraInitContainers }} + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "prometheus-windows-exporter.fullname" . }} + containers: + - name: windows-exporter + image: {{ include "prometheus-windows-exporter.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --config.file=%CONTAINER_SANDBOX_MOUNT_POINT%/config.yml + - --collector.textfile.directories=%CONTAINER_SANDBOX_MOUNT_POINT% + - --web.listen-address=:{{ .Values.service.port }} + {{- with .Values.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + {{- range $key, $value := .Values.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + hostPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + httpGet: + httpHeaders: + {{- range $_, $header := .Values.livenessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ .Values.service.port }} + scheme: {{ upper .Values.livenessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + readinessProbe: + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + httpGet: + httpHeaders: + {{- range $_, $header := .Values.readinessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ .Values.service.port }} + scheme: {{ upper .Values.readinessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: /config.yml + subPath: config.yml + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: true + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + {{- with .Values.sidecars }} + {{- toYaml . | nindent 8 }} + {{- if or .Values.sidecarVolumeMount .Values.sidecarHostVolumeMounts }} + volumeMounts: + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- end }} + {{- range $_, $mount := .Values.sidecarHostVolumeMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- end }} + {{- end }} + {{- end }} + {{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "prometheus-windows-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.imagePullSecrets) | indent 8 }} + {{- end }} + hostNetwork: {{ .Values.hostNetwork }} + hostPID: {{ .Values.hostPID }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.dnsConfig }} + dnsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: exporter-scripts + configMap: + name: {{ include "prometheus-windows-exporter.fullname" . }}-scripts + - name: config + configMap: + name: {{ include "prometheus-windows-exporter.fullname" . }} + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + emptyDir: + medium: Memory + {{- end }} + {{- range $_, $mount := .Values.sidecarHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + configMap: + name: {{ $mount.name }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ $mount.name }} + secret: + secretName: {{ $mount.name }} + {{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/podmonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/podmonitor.yaml new file mode 100644 index 0000000000..bbb6c39340 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/podmonitor.yaml @@ -0,0 +1,91 @@ +{{- if .Values.prometheus.podMonitor.enabled }} +apiVersion: {{ .Values.prometheus.podMonitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: PodMonitor +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.podmonitor-namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.podMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.podMonitor.jobLabel }} + {{- include "podmonitor.scrapeLimits" .Values.prometheus.podMonitor | nindent 2 }} + selector: + matchLabels: + {{- with .Values.prometheus.podMonitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "prometheus-windows-exporter.selectorLabels" . | nindent 6 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ include "prometheus-windows-exporter.namespace" . }} + {{- with .Values.prometheus.podMonitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + podMetricsEndpoints: + - port: {{ .Values.service.portName }} + {{- with .Values.prometheus.podMonitor.scheme }} + scheme: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.path }} + path: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.authorization }} + authorization: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.oauth2 }} + oauth2: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorTimestamps }} + honorTimestamps: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorLabels }} + honorLabels: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + enableHttp2: {{ default false .Values.prometheus.podMonitor.enableHttp2 }} + filterRunning: {{ default true .Values.prometheus.podMonitor.filterRunning }} + followRedirects: {{ default false .Values.prometheus.podMonitor.followRedirects }} + {{- with .Values.prometheus.podMonitor.params }} + params: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/scriptConfig.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/scriptConfig.yaml new file mode 100644 index 0000000000..f514c8161a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/scriptConfig.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }}-scripts + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: +{{ (.Files.Glob "scripts/*").AsConfig | indent 2 }} + diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/service.yaml new file mode 100644 index 0000000000..267b796f63 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/service.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" $ | nindent 4 }} + {{- if or .Values.prometheus.monitor.enabled .Values.prometheus.podMonitor.enabled }} + {{- with .Values.service.annotations }} + annotations: + {{- unset . "prometheus.io/scrape" | toYaml | nindent 4 }} + {{- end }} + {{- else }} + annotations: + prometheus.io/scrape: "true" + {{- with .Values.service.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + {{- if ( and (eq .Values.service.type "NodePort" ) (not (empty .Values.service.nodePort)) ) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: {{ .Values.service.port }} + protocol: TCP + appProtocol: http + name: {{ .Values.service.portName }} + selector: + {{- include "windowsExporter.labels" . | nindent 4 }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/serviceaccount.yaml new file mode 100644 index 0000000000..14c1c46807 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.rbac.create .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "prometheus-windows-exporter.serviceAccountName" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if or .Values.serviceAccount.imagePullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "prometheus-windows-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.serviceAccount.imagePullSecrets) | indent 2 }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/servicemonitor.yaml new file mode 100644 index 0000000000..2effc07758 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/templates/servicemonitor.yaml @@ -0,0 +1,75 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: {{ .Values.prometheus.monitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: ServiceMonitor +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.monitor-namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.monitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.monitor | nindent 2 }} + {{- with .Values.prometheus.monitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- with .Values.prometheus.monitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "windowsExporter.labels" . | nindent 6 }} + {{- end }} + {{- with .Values.prometheus.monitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + endpoints: + - port: {{ .Values.service.portName }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- with .Values.prometheus.monitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + metricRelabelings: +{{- include "windowsExporter.renamedMetricsRelabeling" . | nindent 6 -}} + - sourceLabels: [__name__] + regex: 'wmi_(.*)' + replacement: 'windows_$1' + targetLabel: __name__ + - sourceLabels: [volume, nic] + regex: (.*);(.*) + separator: '' + targetLabel: device + action: replace + replacement: $1$2 + - sourceLabels: [__name__] + regex: windows_cs_logical_processors + replacement: 'system' + targetLabel: mode + relabelings: + - separator: ':' + sourceLabels: + - __meta_kubernetes_pod_host_ip + - __meta_kubernetes_pod_container_port_number + targetLabel: instance +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/values.yaml new file mode 100644 index 0000000000..04569505d6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/charts/windowsExporter/values.yaml @@ -0,0 +1,366 @@ +# Default values for prometheus-windows-exporter. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + registry: docker.io + repository: rancher/mirrored-prometheus-windows-exporter + # Overrides the image tag whose default is {{ printf "v%s" .Chart.AppVersion }} + tag: "0.25.1" + pullPolicy: IfNotPresent + digest: "" + +config: |- + collectors: + enabled: '[defaults],tcp,memory,container' + +imagePullSecrets: [] +# - name: "image-pull-secret" +nameOverride: "" +fullnameOverride: "" + +global: + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + cattle: + systemDefaultRegistry: "" + +service: + type: ClusterIP + port: 9796 + nodePort: + portName: windows-metrics + annotations: {} + +# Additional environment variables that will be passed to the daemonset +env: {} +## env: +## VARIABLE: value + +prometheus: + monitor: + enabled: true + additionalLabels: {} + namespace: "" + + jobLabel: "component" + + # List of pod labels to add to windows exporter metrics + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor + podTargetLabels: ["component"] + + scheme: http + basicAuth: {} + bearerTokenFile: + tlsConfig: {} + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## Override serviceMonitor selector + ## + selectorOverride: {} + + ## Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + ## + attachMetadata: + node: false + + relabelings: [] + metricRelabelings: [] + interval: "" + scrapeTimeout: 10s + ## prometheus.monitor.apiVersion ApiVersion for the serviceMonitor Resource(defaults to "monitoring.coreos.com/v1") + apiVersion: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + # PodMonitor defines monitoring for a set of pods. + # ref. https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.PodMonitor + # Using a PodMonitor may be preferred in some environments where there is very large number + # of Windows Exporter endpoints (1000+) behind a single service. + # The PodMonitor is disabled by default. When switching from ServiceMonitor to PodMonitor, + # the time series resulting from the configuration through PodMonitor may have different labels. + # For instance, there will not be the service label any longer which might + # affect PromQL queries selecting that label. + podMonitor: + enabled: false + # Namespace in which to deploy the pod monitor. Defaults to the release namespace. + namespace: "" + # Additional labels, e.g. setting a label for pod monitor selector as set in prometheus + additionalLabels: {} + # release: kube-prometheus-stack + # PodTargetLabels transfers labels of the Kubernetes Pod onto the target. + podTargetLabels: [] + # apiVersion defaults to monitoring.coreos.com/v1. + apiVersion: "" + # Override pod selector to select pod objects. + selectorOverride: {} + # Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + attachMetadata: + node: false + # The label to use to retrieve the job name from. Defaults to label app.kubernetes.io/name. + jobLabel: "" + + # Scheme/protocol to use for scraping. + scheme: "http" + # Path to scrape metrics at. + path: "/metrics" + + # BasicAuth allow an endpoint to authenticate over basic authentication. + # More info: https://prometheus.io/docs/operating/configuration/#endpoint + basicAuth: {} + # Secret to mount to read bearer token for scraping targets. + # The secret needs to be in the same namespace as the pod monitor and accessible by the Prometheus Operator. + # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secretkeyselector-v1-core + bearerTokenSecret: {} + # TLS configuration to use when scraping the endpoint. + tlsConfig: {} + # Authorization section for this endpoint. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.SafeAuthorization + authorization: {} + # OAuth2 for the URL. Only valid in Prometheus versions 2.27.0 and newer. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.OAuth2 + oauth2: {} + + # ProxyURL eg http://proxyserver:2195. Directs scrapes through proxy to this endpoint. + proxyUrl: "" + # Interval at which endpoints should be scraped. If not specified Prometheus’ global scrape interval is used. + interval: "" + # Timeout after which the scrape is ended. If not specified, the Prometheus global scrape interval is used. + scrapeTimeout: "" + # HonorTimestamps controls whether Prometheus respects the timestamps present in scraped data. + honorTimestamps: true + # HonorLabels chooses the metric’s labels on collisions with target labels. + honorLabels: true + # Whether to enable HTTP2. Default false. + enableHttp2: "" + # Drop pods that are not running. (Failed, Succeeded). + # Enabled by default. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase + filterRunning: "" + # FollowRedirects configures whether scrape requests follow HTTP 3xx redirects. Default false. + followRedirects: "" + # Optional HTTP URL parameters + params: {} + + # RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds + # relabelings for a few standard Kubernetes fields. The original scrape job’s name + # is available via the __tmp_prometheus_job_name label. + # More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + relabelings: [] + # MetricRelabelConfigs to apply to samples before ingestion. + metricRelabelings: [] + + # SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + sampleLimit: 0 + # TargetLimit defines a limit on the number of scraped targets that will be accepted. + targetLimit: 0 + # Per-scrape limit on number of labels that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelLimit: 0 + # Per-scrape limit on length of labels name that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelNameLengthLimit: 0 + # Per-scrape limit on length of labels value that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelValueLengthLimit: 0 + +## Customize the updateStrategy if set +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m +# memory: 30Mi + +serviceAccount: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + annotations: {} + imagePullSecrets: [] + automountServiceAccountToken: false + +securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + +rbac: + ## If true, create & use RBAC resources + ## + create: true + +# Expose the service to the host network +hostNetwork: true + +# Share the host process ID namespace +hostPID: true + +## Assign a group of affinity scheduling rules +## +affinity: {} +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchFields: +# - key: metadata.name +# operator: In +# values: +# - target-host-name + +# Annotations to be added to windows exporter pods +podAnnotations: + # Fix for very slow GKE cluster upgrades + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + +# Extra labels to be added to windows exporter pods +podLabels: {} + +# Annotations to be added to windows exporter daemonset +daemonsetAnnotations: {} + +## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box +releaseLabel: false + +# Custom DNS configuration to be added to prometheus-windows-exporter pods +dnsConfig: {} +# nameservers: +# - 1.2.3.4 +# searches: +# - ns1.svc.cluster-domain.example +# - my.dns.search.suffix +# options: +# - name: ndots +# value: "2" +# - name: edns0 + +## Assign a nodeSelector if operating a hybrid cluster +## +nodeSelector: + kubernetes.io/os: windows + # kubernetes.io/arch: amd64 + +tolerations: + - effect: NoSchedule + operator: Exists + +## Assign a PriorityClassName to pods if set +# priorityClassName: "" + +## Additional container arguments +## +extraArgs: [] +# - --collector.service.services-where +# - "Name LIKE 'sql%'" + +## Additional mounts from the host to windows-exporter container +## +extraHostVolumeMounts: [] +# - name: +# hostPath: +# mountPath: +# readOnly: true|false + +## Additional configmaps to be mounted. +## +configmaps: [] +# - name: +# mountPath: +secrets: [] +# - name: +# mountPath: +## Override the deployment namespace +## +namespaceOverride: "" + +## Additional containers for export metrics to text file +## +sidecars: [] +## - name: nvidia-dcgm-exporter +## image: nvidia/dcgm-exporter:1.4.3 + +## Volume for sidecar containers +## +sidecarVolumeMount: [] +## - name: collector-textfiles +## mountPath: /run/prometheus +## readOnly: false + +## Additional mounts from the host to sidecar containers +## +sidecarHostVolumeMounts: [] +# - name: +# hostPath: +# mountPath: +# readOnly: true|false +# mountPropagation: None|HostToContainer|Bidirectional + +## Additional InitContainers to initialize the pod +## +extraInitContainers: [] + +## Liveness probe +## +livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + +## Readiness probe +## +readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/ingress-nginx/nginx.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/ingress-nginx/nginx.json new file mode 100644 index 0000000000..565352235a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/ingress-nginx/nginx.json @@ -0,0 +1,1445 @@ +{ + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": "$datasource", + "enable": true, + "expr": "sum(changes(nginx_ingress_controller_config_last_reload_successful_timestamp_seconds{instance!=\"unknown\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[30s])) by (controller_class)", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Config Reloads", + "showIn": 0, + "step": "30s", + "tagKeys": "controller_class", + "tags": [], + "titleFormat": "Config Reloaded", + "type": "tags" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1534359654832, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "format": "ops", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 20, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[2m])), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Controller Request Volume", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 82, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(avg_over_time(nginx_ingress_controller_nginx_process_connections{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",state=\"active\"}[2m]))", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Controller Connections", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 80, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": false + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 21, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",status!~\"[4-5].*\"}[2m])) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "95, 99, 99.5", + "title": "Controller Success Rate (non-4|5xx responses)", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 18, + "y": 0 + }, + "id": 81, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg(irate(nginx_ingress_controller_success{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[1m])) * 60", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Config Reloads", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "total" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 0 + }, + "id": 83, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(nginx_ingress_controller_config_last_reload_successful{controller_pod=~\"$controller\",controller_namespace=~\"$namespace\"} == 0)", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Last Config Failed", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 3 + }, + "height": "200px", + "id": 86, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress), 0.001)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "metric": "network", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Ingress Request Volume", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00", + "max - prometheus": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 3 + }, + "id": 87, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 300, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",ingress=~\"$ingress\",status!~\"[4-5].*\"}[2m])) by (ingress) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Ingress Success Rate (non-4|5xx responses)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 1, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "height": "200px", + "id": 32, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (irate (nginx_ingress_controller_request_size_sum{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Received", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "expr": "- sum (irate (nginx_ingress_controller_response_size_sum{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Sent", + "metric": "network", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network I/O pressure", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00", + "max - prometheus": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 77, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(nginx_ingress_controller_nginx_process_resident_memory_bytes{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}) ", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "nginx", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Average Memory Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 3, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 10 + }, + "height": "", + "id": 79, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg (rate (nginx_ingress_controller_nginx_process_cpu_seconds_total{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m])) ", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "nginx", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Average CPU Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": "cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [], + "datasource": "$datasource", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "hideTimeOverride": false, + "id": 75, + "links": [], + "pageSize": 7, + "repeat": null, + "repeatDirection": "h", + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "alias": "Ingress", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "ingress", + "preserveFormat": false, + "sanitize": false, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Requests", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #A", + "thresholds": [ + "" + ], + "type": "number", + "unit": "ops" + }, + { + "alias": "Errors", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "ops" + }, + { + "alias": "P50 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "P90 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "P99 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "IN", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #F", + "thresholds": [ + "" + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "OUT", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #G", + "thresholds": [], + "type": "number", + "unit": "Bps" + } + ], + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "D" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }}", + "refId": "E" + }, + { + "expr": "sum(irate(nginx_ingress_controller_request_size_sum{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "F" + }, + { + "expr": "sum(irate(nginx_ingress_controller_response_size_sum{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "G" + } + ], + "timeFrom": null, + "title": "Ingress Percentile Response Times and Transfer Rates", + "transform": "table", + "transparent": false, + "type": "table" + }, + { + "columns": [ + { + "text": "Current", + "value": "current" + } + ], + "datasource": "$datasource", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "height": "1024", + "id": 85, + "links": [], + "pageSize": 7, + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "TTL", + "colorMode": "cell", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Current", + "thresholds": [ + "0", + "691200" + ], + "type": "number", + "unit": "s" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "avg(nginx_ingress_controller_ssl_expire_time_seconds{kubernetes_pod_name=~\"$controller\",namespace=~\"$namespace\",ingress=~\"$ingress\"}) by (host) - time()", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ host }}", + "metric": "gke_letsencrypt_cert_expiration", + "refId": "A", + "step": 1 + } + ], + "title": "Ingress Certificate Expiry", + "transform": "timeseries_aggregations", + "type": "table" + } + ], + "refresh": "5s", + "schemaVersion": 16, + "style": "dark", + "tags": [ + "nginx" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash, controller_namespace)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Controller Class", + "multi": false, + "name": "controller_class", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash{namespace=~\"$namespace\"}, controller_class) ", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Controller", + "multi": false, + "name": "controller", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash{namespace=~\"$namespace\",controller_class=~\"$controller_class\"}, controller_pod) ", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "tags": [], + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Ingress", + "multi": false, + "name": "ingress", + "options": [], + "query": "label_values(nginx_ingress_controller_requests{namespace=~\"$namespace\",controller_class=~\"$controller_class\",controller_pod=~\"$controller\"}, ingress) ", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "2m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "NGINX / Ingress Controller", + "uid": "nginx", + "version": 1 +} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/ingress-nginx/request-handling-performance.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/ingress-nginx/request-handling-performance.json new file mode 100644 index 0000000000..156e33123d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/ingress-nginx/request-handling-performance.json @@ -0,0 +1,963 @@ +{ + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "gnetId": 9614, + "graphTooltip": 1, + "id": null, + "iteration": 1582146566338, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Total time taken for nginx and upstream servers to process a request and send a response", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 91, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n 0.5,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".5", + "refId": "D" + }, + { + "expr": "histogram_quantile(\n 0.95,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".95", + "refId": "B" + }, + { + "expr": "histogram_quantile(\n 0.99,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".99", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total request handling time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "The time spent on receiving the response from the upstream server", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 94, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n 0.5,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": ".5", + "refId": "D" + }, + { + "expr": "histogram_quantile(\n 0.95,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".95", + "refId": "B" + }, + { + "expr": "histogram_quantile(\n 0.99,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".99", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream response time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 93, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": " sum by (path)(\n rate(\n nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request volume by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "For each path observed, its median upstream response time", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 98, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n .5,\n sum by (le, path)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Median upstream response time by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Percentage of 4xx and 5xx responses among all responses.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 100, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (path) (rate(nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n status =~ \"[4-5].*\"\n}[1m])) / sum by (path) (rate(nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n}[1m]))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response error rate by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "For each path observed, the sum of upstream request time", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 102, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (path) (rate(nginx_ingress_controller_response_duration_seconds_sum{ingress =~ \"$ingress\"}[1m]))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream time consumed by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 101, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": " sum (\n rate(\n nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n status =~\"[4-5].*\",\n }[1m]\n )\n ) by(path, status)\n", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }} {{ status }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response error volume by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "hiddenSeries": false, + "id": 99, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (\n rate (\n nginx_ingress_controller_response_size_sum {\n ingress =~ \"$ingress\",\n }[1m]\n )\n) by (path) / sum (\n rate(\n nginx_ingress_controller_response_size_count {\n ingress =~ \"$ingress\",\n }[1m]\n )\n) by (path)\n", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "D" + }, + { + "expr": " sum (rate(nginx_ingress_controller_response_size_bucket{\n ingress =~ \"$ingress\",\n }[1m])) by (le)\n", + "hide": true, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Average response size by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 96, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (\n rate(\n nginx_ingress_controller_ingress_upstream_latency_seconds_sum {\n ingress =~ \"$ingress\",\n }[1m]\n)) / sum (\n rate(\n nginx_ingress_controller_ingress_upstream_latency_seconds_count {\n ingress =~ \"$ingress\",\n }[1m]\n )\n)\n", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "average", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream service latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 22, + "style": "dark", + "tags": [ + "nginx" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": {}, + "datasource": "$datasource", + "definition": "label_values(nginx_ingress_controller_requests, ingress) ", + "hide": 0, + "includeAll": true, + "label": "Service Ingress", + "multi": false, + "name": "ingress", + "options": [], + "query": "label_values(nginx_ingress_controller_requests, ingress) ", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "2m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "NGINX / Request Handling Performance", + "uid": "4GFbkOsZk", + "version": 1 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/cluster/rancher-cluster-nodes.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/cluster/rancher-cluster-nodes.json new file mode 100644 index 0000000000..1d4943501b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/cluster/rancher-cluster-nodes.json @@ -0,0 +1,793 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m] ({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5 OR avg_over_time(windows_system_processor_queue_length[5m])) by (instance)", + "interval": "", + "legendFormat": "Load[5m] ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(node_load1 OR avg_over_time(windows_system_processor_queue_length[1m])) by (instance)", + "interval": "", + "legendFormat": "Load[1m] ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(node_load15 OR avg_over_time(windows_system_processor_queue_length[15m])) by (instance)", + "interval": "", + "legendFormat": "Load[15m] ({{instance}})", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(node_memory_MemAvailable_bytes OR windows_os_physical_memory_free_bytes) by (instance) / sum(node_memory_MemTotal_bytes OR windows_cs_physical_memory_bytes) by (instance) ", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance))", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Read ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Write ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Errors ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Total ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Errors ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Dropped ({{instance}})", + "refId": "D" + }, + { + "expr": "sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Dropped ({{instance}})", + "refId": "E" + }, + { + "expr": "sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Total ({{instance}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Total ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Total ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Cluster (Nodes)", + "uid": "rancher-cluster-nodes-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/cluster/rancher-cluster.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/cluster/rancher-cluster.json new file mode 100644 index 0000000000..24385a237a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/cluster/rancher-cluster.json @@ -0,0 +1,776 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[$__rate_interval]))", + "legendFormat": "Total", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5 OR avg_over_time(windows_system_processor_queue_length[5m]))", + "interval": "", + "legendFormat": "Load[5m]", + "refId": "A" + }, + { + "expr": "sum(node_load1 OR avg_over_time(windows_system_processor_queue_length[1m]))", + "interval": "", + "legendFormat": "Load[1m]", + "refId": "B" + }, + { + "expr": "sum(node_load15 OR avg_over_time(windows_system_processor_queue_length[15m]))", + "interval": "", + "legendFormat": "Load[15m]", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(node_memory_MemAvailable_bytes OR windows_os_physical_memory_free_bytes) / sum(node_memory_MemTotal_bytes OR windows_cs_physical_memory_bytes)", + "legendFormat": "Total", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}))", + "legendFormat": "Total", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Read", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Write", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "A" + }, + { + "expr": "(sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + }, + { + "expr": "(sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "C" + }, + { + "expr": "(sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "D" + }, + { + "expr": "(sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "E" + }, + { + "expr": "(sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Cluster", + "uid": "rancher-cluster-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/bundle.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/bundle.json new file mode 100644 index 0000000000..698f48aeed --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/bundle.json @@ -0,0 +1,246 @@ +{ + "description": "Bundle", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_out_of_sync{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Out of Sync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_err_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_pending{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_wait_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_out_of_sync{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Out of Sync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_err_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_pending{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_wait_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Bundles", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_bundle_desired_ready, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_bundle_desired_ready{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / Bundle", + "uid": "fleet-bundle" +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/bundledeployment.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/bundledeployment.json new file mode 100644 index 0000000000..c81f7a6212 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/bundledeployment.json @@ -0,0 +1,219 @@ +{ + "description": "BundleDeployment", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Ready\"}) / sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\"})" + } + ], + "title": "Ready BundleDeployments", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"WaitApplied\"})", + "legendFormat": "Wait Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"ErrApplied\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"OutOfSync\"})", + "legendFormat": "OutOfSync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Pending\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Modified\"})", + "legendFormat": "Modified" + } + ], + "title": "BundleDeployments", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"WaitApplied\"})", + "legendFormat": "Wait Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"ErrApplied\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"OutOfSync\"})", + "legendFormat": "OutOfSync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Pending\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Modified\"})", + "legendFormat": "Modified" + } + ], + "title": "BundleDeployments", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_bundledeployment_state, cluster_namespace)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / BundleDeployment", + "uid": "fleet-bundledeployment" +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/cluster.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/cluster.json new file mode 100644 index 0000000000..73bdea4834 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/cluster.json @@ -0,0 +1,484 @@ +{ + "description": "Cluster", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_desired_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Git Repos", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_desired_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Git Repos", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_desired_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Git Repos", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 13 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_resources_count_desiredready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 13 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_desiredready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_desiredready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 26 + }, + "id": 7, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"Ready\"}) / sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 26 + }, + "id": 8, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"WaitCheckIn\"})", + "legendFormat": "Wait Check In" + } + ], + "title": "Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 9, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"WaitCheckIn\"})", + "legendFormat": "Wait Check In" + } + ], + "title": "Clusters", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_cluster_desired_ready_git_repos, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_cluster_desired_ready_git_repos{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / Cluster", + "uid": "fleet-cluster" +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/clustergroup.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/clustergroup.json new file mode 100644 index 0000000000..ce3df87b21 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/clustergroup.json @@ -0,0 +1,468 @@ +{ + "description": "ClusterGroup", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_group_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Bundles", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 13 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "(sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"}) - sum(fleet_cluster_group_non_ready_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})) / sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 13 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Total" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_non_ready_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Non Ready" + } + ], + "title": "Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Total" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_non_ready_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Non Ready" + } + ], + "title": "Clusters", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 26 + }, + "id": 7, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_group_resource_count_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 26 + }, + "id": 8, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 9, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_cluster_group_bundle_desired_ready, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_cluster_group_bundle_desired_ready{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / ClusterGroup", + "uid": "fleet-cluster-group" +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/controller-runtime.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/controller-runtime.json new file mode 100644 index 0000000000..23a81f2a8c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/controller-runtime.json @@ -0,0 +1,454 @@ +{ + "description": "Controller Runtime", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "controller_runtime_active_workers{job=\"$job\", namespace=\"$namespace\"}", + "legendFormat": "{{controller}} {{instance}}" + } + ], + "title": "Number of Workers in Use", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(controller_runtime_reconcile_errors_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, pod)", + "legendFormat": "{{instance}} {{pod}}" + } + ], + "title": "Reconciliation Error Count per Controller", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(controller_runtime_reconcile_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, pod)", + "legendFormat": "{{instance}} {{pod}}" + } + ], + "title": "Total Reconciliation Count per Controller", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "workqueue_depth{job=\"$job\", namespace=\"$namespace\"}", + "legendFormat": "{{instance}} {{pod}}" + } + ], + "title": "WorkQueue Depth", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P50 {{name}}" + } + ], + "title": "Seconds for Items Stay in Queue (before being requested) P50", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.90, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P90 {{name}}" + } + ], + "title": "Seconds for Items Stay in Queue (before being requested) P90", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 7, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P99 {{name}}" + } + ], + "title": "Seconds for Items Stay in Queue (before being requested) P99", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 8, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(workqueue_adds_total{job=\"$job\", namespace=\"$namespace\"}[2m])) by (instance, name)", + "legendFormat": "{{name}} {{instance}}" + } + ], + "title": "Work Queue Add Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 9, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(workqueue_unfinished_work_seconds{job=\"$job\", namespace=\"$namespace\"}[5m])", + "legendFormat": "{{name}} {{instance}}" + } + ], + "title": "Unfinished Seconds", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 72 + }, + "id": 10, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P50 {{name}}" + } + ], + "title": "Seconds Processing Items from WorkQueue - 50th Percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 80 + }, + "id": 11, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.90, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P90 {{name}}" + } + ], + "title": "Seconds Processing Items from WorkQueue - 90th Percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 88 + }, + "id": 12, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P99 {{name}}" + } + ], + "title": "Seconds Processing Items from WorkQueue - 99th Percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 96 + }, + "id": 13, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(workqueue_retries_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name)", + "legendFormat": "{{name}} {{instance}}" + } + ], + "title": "Work Queue Retries Rate", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(controller_runtime_reconcile_total, namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "job", + "query": "label_values(controller_runtime_reconcile_total{namespace=~\"$namespace\"}, job)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / Controller-Runtime", + "uid": "fleet-controller-runtime" +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/gitrepo.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/gitrepo.json new file mode 100644 index 0000000000..1a50c2937d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/fleet/gitrepo.json @@ -0,0 +1,325 @@ +{ + "description": "GitRepo", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_gitrepo_desired_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_desired_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_desired_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Clusters", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 13 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_gitrepo_resources_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 13 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + } + ], + "title": "Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + } + ], + "title": "Resources", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_gitrepo_desired_ready_clusters, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_gitrepo_desired_ready_clusters{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / GitRepo", + "uid": "fleet-gitrepo" +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/home/rancher-default-home.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/home/rancher-default-home.json new file mode 100644 index 0000000000..3fce207561 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/home/rancher-default-home.json @@ -0,0 +1,1290 @@ +{ + "annotations": { + "list": [] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "title": "", + "type": "welcome" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 4 + }, + "height": "180px", + "id": 6, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - (avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[5m])))) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "CPU Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 4 + }, + "height": "180px", + "id": 4, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"}) / sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"})) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Memory Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 4 + }, + "height": "180px", + "id": 7, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - (((sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))) / ((sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))))) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Disk Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 9 + }, + "height": "1px", + "id": 11, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode!=\"idle\"}[5m]))", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "CPU Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 9 + }, + "height": "1px", + "id": 12, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_node_status_allocatable_cpu_cores{}) OR sum(kube_node_status_allocatable{resource=\"cpu\",unit=\"core\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "CPU Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 9 + }, + "height": "1px", + "id": 9, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "20%", + "prefix": "", + "prefixFontSize": "20%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"}) - sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Memory Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 9 + }, + "height": "1px", + "id": 10, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_node_status_allocatable_memory_bytes{}) OR sum(kube_node_status_allocatable{resource=\"memory\", unit=\"byte\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Memory Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 9 + }, + "height": "1px", + "id": 13, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) - sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) - sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Disk Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 9 + }, + "height": "1px", + "id": 14, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Disk Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 2051, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[$__rate_interval])))", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\", mode=\"idle\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 12 + }, + "hiddenSeries": false, + "id": 2052, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100 * (1 - sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"}) / sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"}))", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "100 * (1- sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"}) by (instance) / sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"}) by (instance))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 12 + }, + "hiddenSeries": false, + "id": 2053, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(1 - ((sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"} OR on() vector(0)))) / ((sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0)))) * 100", + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "(1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) by (instance)) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) by (instance)) * 100", + "hide": false, + "legendFormat": "{{ instance }}", + "refId": "B" + }, + { + "expr": "(1 - (sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance)) / sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance)) * 100", + "hide": false, + "legendFormat": "{{ instance }}", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "folderId": 0, + "gridPos": { + "h": 15, + "w": 12, + "x": 0, + "y": 18 + }, + "headings": true, + "id": 3, + "limit": 30, + "links": [], + "query": "", + "recent": true, + "search": true, + "starred": false, + "tags": [], + "title": "Dashboards", + "type": "dashlist" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 2055, + "options": { + "content": "## About Rancher Monitoring\n\nRancher Monitoring is a Helm chart developed by Rancher that is powered by [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator). It is based on the upstream [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) Helm chart maintained by the Prometheus community.\n\nBy default, the chart deploys Grafana alongside a set of Grafana dashboards curated by the [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) project.\n\nFor more information on how Rancher Monitoring differs from [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack), please view the CHANGELOG.md of the rancher-monitoring chart located in the [rancher/charts](https://github.com/rancher/charts) repository.\n\nFor more information about how to configure Rancher Monitoring, please view the [Rancher docs](https://rancher.com/docs/rancher/v2.x/en/).\n\n", + "mode": "markdown" + }, + "pluginVersion": "7.1.0", + "timeFrom": null, + "timeShift": null, + "title": "", + "type": "text" + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "hidden": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "Home", + "uid": "rancher-home-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-etcd-nodes.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-etcd-nodes.json new file mode 100644 index 0000000000..8af4b81ce0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-etcd-nodes.json @@ -0,0 +1,687 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 32, + "links": [], + "panels": [ + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_client_grpc_received_bytes_total{job=\"kube-etcd\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Client Traffic In ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_network_client_grpc_sent_bytes_total{job=\"kube-etcd\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Client Traffic Out ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GRPC Client Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(etcd_mvcc_db_total_size_in_bytes) by (instance)", + "interval": "", + "legendFormat": "DB Size ({{instance}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DB Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) by (instance) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) by (instance)", + "interval": "", + "legendFormat": "Watch Streams ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) by (instance) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) by (instance)", + "interval": "", + "legendFormat": "Lease Watch Stream ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_committed_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Proposal Committed ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Proposal Applied ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(rate(etcd_server_proposals_failed_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Proposal Failed ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(etcd_server_proposals_pending) by (instance)", + "interval": "", + "legendFormat": "Proposal Pending ({{instance}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{grpc_type=\"unary\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "RPC Rate ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(grpc_server_handled_total{grpc_type=\"unary\",grpc_code!=\"OK\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "RPC Failure Rate ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "WAL fsync ({{instance}})", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "DB fsync ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / etcd (Nodes)", + "uid": "rancher-etcd-nodes-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-etcd.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-etcd.json new file mode 100644 index 0000000000..0c058cafb9 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-etcd.json @@ -0,0 +1,669 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 33, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_client_grpc_received_bytes_total{job=\"kube-etcd\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Client Traffic In", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_network_client_grpc_sent_bytes_total{job=\"kube-etcd\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Client Traffic Out", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GRPC Client Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(etcd_mvcc_db_total_size_in_bytes)", + "interval": "", + "legendFormat": "DB Size", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DB Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", + "interval": "", + "legendFormat": "Watch Streams", + "refId": "A" + }, + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", + "interval": "", + "legendFormat": "Lease Watch Stream", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_committed_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Proposal Committed", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Proposal Applied", + "refId": "B" + }, + { + "expr": "sum(rate(etcd_server_proposals_failed_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Proposal Failed", + "refId": "C" + }, + { + "expr": "sum(etcd_server_proposals_pending)", + "interval": "", + "legendFormat": "Proposal Pending", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{grpc_type=\"unary\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "RPC Rate", + "refId": "A" + }, + { + "expr": "sum(rate(grpc_server_handled_total{grpc_type=\"unary\",grpc_code!=\"OK\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "RPC Failure Rate", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "WAL fsync", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "DB fsync", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / etcd", + "uid": "rancher-etcd-1", + "version": 4 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-k8s-components-nodes.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-k8s-components-nodes.json new file mode 100644 index 0000000000..b31358eaaf --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-k8s-components-nodes.json @@ -0,0 +1,527 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 30, + "links": [], + "panels": [ + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(apiserver_request_total[$__rate_interval])) by (instance, code)", + "interval": "", + "legendFormat": "{{code}}({{instance}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API Server Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"deployment\"}) by (instance, name)", + "interval": "", + "legendFormat": "Deployment Depth ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"volumes\"}) by (instance, name)", + "interval": "", + "legendFormat": "Volumes Depth ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicaset\"}) by (instance, name)", + "interval": "", + "legendFormat": "ReplicaSet Depth ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"service\"}) by (instance, name)", + "interval": "", + "legendFormat": "Service Depth ({{instance}})", + "refId": "D" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"serviceaccount\"}) by (instance, name)", + "interval": "", + "legendFormat": "ServiceAccount Depth ({{instance}})", + "refId": "E" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"endpoint\"}) by (instance, name)", + "interval": "", + "legendFormat": "Endpoint Depth ({{instance}})", + "refId": "F" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"daemonset\"}) by (instance, name)", + "interval": "", + "legendFormat": "DaemonSet Depth ({{instance}})", + "refId": "G" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"statefulset\"}) by (instance, name)", + "interval": "", + "legendFormat": "StatefulSet Depth ({{instance}})", + "refId": "H" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicationmanager\"}) by (instance, name)", + "interval": "", + "legendFormat": "ReplicationManager Depth ({{instance}})", + "refId": "I" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Controller Manager Queue Depth", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_status_scheduled{condition=\"false\"})", + "interval": "", + "legendFormat": "Failed To Schedule", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pod Scheduling Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"reading\"}) by (instance)", + "interval": "", + "legendFormat": "Reading ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"waiting\"}) by (instance)", + "interval": "", + "legendFormat": "Waiting ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"writing\"}) by (instance)", + "interval": "", + "legendFormat": "Writing ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"accepted\"}[$__rate_interval]))) by (instance)", + "interval": "", + "legendFormat": "Accepted ({{instance}})", + "refId": "D" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"handled\"}[$__rate_interval]))) by (instance)", + "interval": "", + "legendFormat": "Handled ({{instance}})", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Ingress Controller Connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Kubernetes Components (Nodes)", + "uid": "rancher-k8s-components-nodes-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-k8s-components.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-k8s-components.json new file mode 100644 index 0000000000..44cf97f9fd --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/k8s/rancher-k8s-components.json @@ -0,0 +1,519 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 31, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(apiserver_request_total[$__rate_interval])) by (code)", + "interval": "", + "legendFormat": "{{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API Server Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"deployment\"}) by (name)", + "interval": "", + "legendFormat": "Deployment Depth", + "refId": "A" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"volumes\"}) by (name)", + "interval": "", + "legendFormat": "Volumes Depth", + "refId": "B" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicaset\"}) by (name)", + "interval": "", + "legendFormat": "Replicaset Depth", + "refId": "C" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"service\"}) by (name)", + "interval": "", + "legendFormat": "Service Depth", + "refId": "D" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"serviceaccount\"}) by (name)", + "interval": "", + "legendFormat": "ServiceAccount Depth", + "refId": "E" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"endpoint\"}) by (name)", + "interval": "", + "legendFormat": "Endpoint Depth", + "refId": "F" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"daemonset\"}) by (name)", + "interval": "", + "legendFormat": "DaemonSet Depth", + "refId": "G" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"statefulset\"}) by (name)", + "interval": "", + "legendFormat": "StatefulSet Depth", + "refId": "H" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicationmanager\"}) by (name)", + "interval": "", + "legendFormat": "ReplicationManager Depth", + "refId": "I" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Controller Manager Queue Depth", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_status_scheduled{condition=\"false\"})", + "interval": "", + "legendFormat": "Failed To Schedule", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pod Scheduling Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"reading\"})", + "interval": "", + "legendFormat": "Reading", + "refId": "A" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"waiting\"})", + "interval": "", + "legendFormat": "Waiting", + "refId": "B" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"writing\"})", + "interval": "", + "legendFormat": "Writing", + "refId": "C" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"accepted\"}[$__rate_interval])))", + "interval": "", + "legendFormat": "Accepted", + "refId": "D" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"handled\"}[$__rate_interval])))", + "interval": "", + "legendFormat": "Handled", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Ingress Controller Connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Kubernetes Components", + "uid": "rancher-k8s-components-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/nodes/rancher-node-detail.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/nodes/rancher-node-detail.json new file mode 100644 index 0000000000..920fb94cf7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/nodes/rancher-node-detail.json @@ -0,0 +1,805 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": { + "{{mode}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\", instance=\"$instance\"}[$__rate_interval])) by (mode)", + "interval": "", + "legendFormat": "{{mode}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[5m]))", + "interval": "", + "legendFormat": "Load[5m]", + "refId": "A" + }, + { + "expr": "sum(node_load1{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[1m]))", + "interval": "", + "legendFormat": "Load[1m]", + "refId": "B" + }, + { + "expr": "sum(node_load15{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[15m]))", + "interval": "", + "legendFormat": "Load[15m]", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (node_memory_MemAvailable_bytes{instance=~\"$instance\"} OR windows_os_physical_memory_free_bytes{instance=~\"$instance\"}) / (node_memory_MemTotal_bytes{instance=~\"$instance\"} OR windows_cs_physical_memory_bytes{instance=~\"$instance\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}) by (device) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}) by (device))", + "interval": "", + "legendFormat": "{{device}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Read ({{device}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total{instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Write ({{device}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Errors ({{device}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Total ({{device}})", + "refId": "B" + }, + { + "expr": "sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Errors ({{device}})", + "refId": "C" + }, + { + "expr": "sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Dropped ({{device}})", + "refId": "D" + }, + { + "expr": "sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Dropped ({{device}})", + "refId": "E" + }, + { + "expr": "sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Total ({{device}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Total ({{device}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Total ({{device}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "query": "label_values({__name__=~\"node_exporter_build_info|windows_exporter_build_info\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Node (Detail)", + "uid": "rancher-node-detail-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/nodes/rancher-node.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/nodes/rancher-node.json new file mode 100644 index 0000000000..367df3cc9d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/nodes/rancher-node.json @@ -0,0 +1,792 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\", instance=\"$instance\", mode=\"idle\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[5m]))", + "interval": "", + "legendFormat": "Load[5m]", + "refId": "A" + }, + { + "expr": "sum(node_load1{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[1m]))", + "interval": "", + "legendFormat": "Load[1m]", + "refId": "B" + }, + { + "expr": "sum(node_load15{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[15m]))", + "interval": "", + "legendFormat": "Load[15m]", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(node_memory_MemAvailable_bytes{instance=~\"$instance\"} OR windows_os_physical_memory_free_bytes{instance=~\"$instance\"}) / sum(node_memory_MemTotal_bytes{instance=~\"$instance\"} OR windows_cs_physical_memory_bytes{instance=~\"$instance\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}))", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Read", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total{instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Write", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "A" + }, + { + "expr": "(sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + }, + { + "expr": "(sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "C" + }, + { + "expr": "(sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "D" + }, + { + "expr": "(sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "E" + }, + { + "expr": "(sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "query": "label_values({__name__=~\"node_exporter_build_info|windows_exporter_build_info\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Node", + "uid": "rancher-node-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/performance/performance-debugging.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/performance/performance-debugging.json new file mode 100644 index 0000000000..454bc39390 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/performance/performance-debugging.json @@ -0,0 +1,1652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name) (rate(lasso_controller_reconcile_time_seconds_sum[5m]))\n/\nsum by (handler_name) (rate(lasso_controller_reconcile_time_seconds_count[5m])))", + "interval": "", + "legendFormat": "{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Handler Average Execution Times Over Last 5 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1390", + "format": "short", + "label": "Execution Time in Seconds", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1391", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (resource, method, code) (rate(steve_api_request_time_sum{resource!=\"subscribe\"}[5m]))\n/\nsum by (resource, method, code) (rate(steve_api_request_time_count{resource!=\"subscribe\"}[5m])))", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Rancher API Average Request Times Over Last 5 Minutes (Top 20) (Subscribes Omitted)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:178", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:179", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 30, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "rate(steve_api_request_time_sum{resource=\"subscribe\"}[5m])\n/\nrate(steve_api_request_time_count{resource=\"subscribe\"}[5m])", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Subscribe Average Request Times Over Last 5 Minutes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:368", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:369", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,workqueue_depth)", + "interval": "", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Lasso Controller Work Queue Depth (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1553", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1554", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 13, + "w": 16, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (id, resource, method, code) (steve_api_total_requests))", + "instant": false, + "interval": "", + "legendFormat": "{{id}} {{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Number of Rancher Requests (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:290", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:291", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 16, + "x": 0, + "y": 45 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (id, resource, method) (steve_api_total_requests{code!=\"200\",code!=\"201\"}))", + "interval": "", + "legendFormat": "{{id}} {{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Number of Failed Rancher API Requests (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:428", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:429", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 54 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (resource, method, code) (rate(k8s_proxy_store_request_time_sum[5m]))\n/\nsum by (resource, method, code) (rate(k8s_proxy_store_request_time_count[5m])))", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "K8s Proxy Store Average Request Times Over Last 5 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:662", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:663", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 62 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (resource, method, code) (rate(k8s_proxy_client_request_time_sum[5m]))\n/\nsum by (resource, method, code) (rate(k8s_proxy_client_request_time_count[5m])))", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "K8s Proxy Client Average Request Times Over Last 5 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1710", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1711", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 70 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,lasso_controller_total_cached_object)", + "interval": "", + "legendFormat": "{{kind}} {{version}} {{group}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Cached Objects by GroupVersionKind (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:744", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:745", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 78 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name) (\nlasso_controller_total_handler_execution\n))", + "interval": "", + "legendFormat": "{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Lasso Handler Executions (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:824", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:825", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 86 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20, sum by (handler_name,controller_name) (\nincrease(lasso_controller_total_handler_execution[2m])\n))", + "interval": "", + "legendFormat": "{{controller_name}}.{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Handler Executions Over Last 2 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 94 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name) (\nlasso_controller_total_handler_execution{has_error=\"true\"}\n))", + "interval": "", + "legendFormat": "{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total Handler Executions with Error (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1230", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1231", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 102 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name,controller_name) (\nincrease(lasso_controller_total_handler_execution{has_error=\"true\"}[2m])\n))", + "interval": "", + "legendFormat": "{{controller_name}}.{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Handler Executions Over Last 2 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 110 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,session_server_total_transmit_bytes)", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Data Transmitted by Remote Dialer Sessions (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1953", + "format": "decbytes", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1954", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 118 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "session_server_total_transmit_error_bytes", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Errors for Remote Dialer Sessions (Top 20)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2045", + "format": "ms", + "label": "Error Data", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2046", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 126 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "session_server_total_add_websocket_session - (session_server_total_remove_websocket_session or (0 * session_server_total_add_websocket_session))", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Remote Dialer Active Connections (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2199", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2200", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 134 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(session_server_total_remove_connections[$__rate_interval])", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Remote Dialer Removed Connections Rate (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2199", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2200", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 142 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(session_server_total_add_connections[$__rate_interval])", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Remote Dialer Added Connections Rate (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2117", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2118", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Rancher Performance Debugging", + "uid": "tfrfU0a7k", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/pods/rancher-pod-containers.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/pods/rancher-pod-containers.json new file mode 100644 index 0000000000..cf78a2204c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/pods/rancher-pod-containers.json @@ -0,0 +1,636 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_cfs_throttled_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "CFS throttled ({{container}})", + "refId": "A" + }, + { + "expr": "sum(rate(container_cpu_system_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "System ({{container}})", + "refId": "B" + }, + { + "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Total ({{container}})", + "refId": "C" + }, + { + "expr": "sum(rate(container_cpu_user_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_usermode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "User ({{container}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"} OR windows_container_memory_usage_commit_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}) by (container)", + "interval": "", + "legendFormat": "({{container}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Total ({{container}})", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Total ({{container}})", + "refId": "B" + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Dropped ({{container}})", + "refId": "C" + }, + { + "expr": "sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Errors ({{container}})", + "refId": "D" + }, + { + "expr": "sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Errors ({{container}})", + "refId": "E" + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Dropped ({{container}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Total ({{container}})", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Total ({{container}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Write ({{container}})", + "refId": "A" + }, + { + "expr": "sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Read ({{container}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "label_values(kube_pod_info{}, namespace)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "query": "label_values(kube_pod_info{namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Pod (Containers)", + "uid": "rancher-pod-containers-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/pods/rancher-pod.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/pods/rancher-pod.json new file mode 100644 index 0000000000..4859eccc74 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/pods/rancher-pod.json @@ -0,0 +1,636 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_cfs_throttled_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "CFS throttled", + "refId": "A" + }, + { + "expr": "sum(rate(container_cpu_system_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "System", + "refId": "B" + }, + { + "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Total", + "refId": "C" + }, + { + "expr": "sum(rate(container_cpu_user_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_usermode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "User", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"} OR windows_container_memory_usage_commit_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "C" + }, + { + "expr": "sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "D" + }, + { + "expr": "sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "E" + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Write", + "refId": "A" + }, + { + "expr": "sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Read", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "label_values(kube_pod_info{}, namespace)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "query": "label_values(kube_pod_info{namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Pod", + "uid": "rancher-pod-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/workloads/rancher-workload-pods.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/workloads/rancher-workload-pods.json new file mode 100644 index 0000000000..92c0d24a6e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/workloads/rancher-workload-pods.json @@ -0,0 +1,652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(container_cpu_cfs_throttled_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "CFS throttled ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(rate(container_cpu_system_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "System ({{pod}})", + "refId": "B" + }, + { + "expr": "(sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Total ({{pod}})", + "refId": "C" + }, + { + "expr": "(sum(rate(container_cpu_user_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_usermode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "User ({{pod}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(container_memory_working_set_bytes{namespace=~\"$namespace\",container=\"\"} OR windows_container_memory_usage_commit_bytes{namespace=~\"$namespace\"}) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "({{pod}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Total ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Total ({{pod}})", + "refId": "B" + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Dropped ({{pod}})", + "refId": "C" + }, + { + "expr": "(sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Errors ({{pod}})", + "refId": "D" + }, + { + "expr": "(sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Errors ({{pod}})", + "refId": "E" + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Dropped ({{pod}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Total ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Total ({{pod}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Write ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Read ({{pod}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "query_result(kube_pod_info{namespace!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*namespace=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "kind", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_kind=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind=\"$kind\", created_by_name!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_name=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Workload (Pods)", + "uid": "rancher-workload-pods-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/workloads/rancher-workload.json b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/workloads/rancher-workload.json new file mode 100644 index 0000000000..9f5317c2f0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/rancher/workloads/rancher-workload.json @@ -0,0 +1,652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(rate(container_cpu_cfs_throttled_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "CFS throttled", + "refId": "A" + }, + { + "expr": "sum((sum(rate(container_cpu_system_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "System", + "refId": "B" + }, + { + "expr": "sum((sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Total", + "refId": "C" + }, + { + "expr": "sum((sum(rate(container_cpu_user_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_usermode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "User", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(container_memory_working_set_bytes{namespace=~\"$namespace\"} OR windows_container_memory_usage_commit_bytes{namespace=~\"$namespace\"}) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum((sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + }, + { + "expr": "sum((sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "C" + }, + { + "expr": "sum((sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "D" + }, + { + "expr": "sum((sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "E" + }, + { + "expr": "sum((sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum((sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Write", + "refId": "A" + }, + { + "expr": "sum((sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Read", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "query_result(kube_pod_info{namespace!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*namespace=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "kind", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_kind=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind=\"$kind\", created_by_name!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_name=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Workload", + "uid": "rancher-workload-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/upgrade/scripts/delete-workloads-with-old-labels.sh b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/upgrade/scripts/delete-workloads-with-old-labels.sh new file mode 100644 index 0000000000..89431e7132 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/files/upgrade/scripts/delete-workloads-with-old-labels.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e +set -x + +# node-exporter +kubectl delete daemonset -l app=prometheus-node-exporter,release=rancher-monitoring --ignore-not-found=true + +# prometheus-adapter +kubectl delete deployments -l app=prometheus-adapter,release=rancher-monitoring --ignore-not-found=true + +# kube-state-metrics +kubectl delete deployments -l app.kubernetes.io/instance=rancher-monitoring,app.kubernetes.io/name=kube-state-metrics --cascade=orphan --ignore-not-found=true +kubectl delete statefulsets -l app.kubernetes.io/instance=rancher-monitoring,app.kubernetes.io/name=kube-state-metrics --cascade=orphan --ignore-not-found=true diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/NOTES.txt b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/NOTES.txt new file mode 100644 index 0000000000..371f3ae398 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/NOTES.txt @@ -0,0 +1,4 @@ +{{ $.Chart.Name }} has been installed. Check its status by running: + kubectl --namespace {{ template "kube-prometheus-stack.namespace" . }} get pods -l "release={{ $.Release.Name }}" + +Visit https://github.com/prometheus-operator/kube-prometheus for instructions on how to create & configure Alertmanager and Prometheus instances using the Operator. diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/_helpers.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/_helpers.tpl new file mode 100644 index 0000000000..d2207dd90d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/_helpers.tpl @@ -0,0 +1,459 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{- define "monitoring_registry" -}} + {{- $temp_registry := (include "system_default_registry" .) -}} + {{- if $temp_registry -}} + {{- trimSuffix "/" $temp_registry -}} + {{- else -}} + {{- .Values.global.imageRegistry -}} + {{- end -}} +{{- end -}} + +{{/* +https://github.com/helm/helm/issues/4535#issuecomment-477778391 +Usage: {{ include "call-nested" (list . "SUBCHART_NAME" "TEMPLATE") }} +e.g. {{ include "call-nested" (list . "grafana" "grafana.fullname") }} +*/}} +{{- define "call-nested" }} +{{- $dot := index . 0 }} +{{- $subchart := index . 1 | splitList "." }} +{{- $template := index . 2 }} +{{- $values := $dot.Values }} +{{- range $subchart }} +{{- $values = index $values . }} +{{- end }} +{{- include $template (dict "Chart" (dict "Name" (last $subchart)) "Values" $values "Release" $dot.Release "Capabilities" $dot.Capabilities) }} +{{- end }} + +# Special Exporters +{{- define "exporter.kubeEtcd.enabled" -}} +{{- if or .Values.kubeEtcd.enabled .Values.rkeEtcd.enabled .Values.kubeAdmEtcd.enabled .Values.rke2Etcd.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeControllerManager.enabled" -}} +{{- if or .Values.kubeControllerManager.enabled .Values.rkeControllerManager.enabled .Values.k3sServer.enabled .Values.kubeAdmControllerManager.enabled .Values.rke2ControllerManager.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeScheduler.enabled" -}} +{{- if or .Values.kubeScheduler.enabled .Values.rkeScheduler.enabled .Values.k3sServer.enabled .Values.kubeAdmScheduler.enabled .Values.rke2Scheduler.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeProxy.enabled" -}} +{{- if or .Values.kubeProxy.enabled .Values.rkeProxy.enabled .Values.k3sServer.enabled .Values.kubeAdmProxy.enabled .Values.rke2Proxy.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubelet.enabled" -}} +{{- if or .Values.kubelet.enabled .Values.hardenedKubelet.enabled .Values.k3sServer.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeControllerManager.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-controller-manager +{{- end -}} +{{- end }} + +{{- define "exporter.kubeScheduler.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-scheduler +{{- end -}} +{{- end }} + +{{- define "exporter.kubeProxy.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-proxy +{{- end -}} +{{- end }} + +{{- define "exporter.kubelet.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kubelet +{{- end -}} +{{- end }} + +{{- define "kubelet.serviceMonitor.resourcePath" -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if not (eq .Values.kubelet.serviceMonitor.resourcePath "/metrics/resource/v1alpha1") -}} +{{ .Values.kubelet.serviceMonitor.resourcePath }} +{{- else if semverCompare ">=1.20.0-0" $kubeTargetVersion -}} +/metrics/resource +{{- else -}} +/metrics/resource/v1alpha1 +{{- end -}} +{{- end }} + +{{- define "rancher.serviceMonitor.selector" -}} +{{- if .Values.rancherMonitoring.selector }} +{{ .Values.rancherMonitoring.selector | toYaml }} +{{- else }} +{{- $rancherDeployment := (lookup "apps/v1" "Deployment" "cattle-system" "rancher") }} +{{- if $rancherDeployment }} +matchLabels: + app: rancher + chart: {{ index $rancherDeployment.metadata.labels "chart" }} + release: rancher +{{- end }} +{{- end }} +{{- end }} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# Prometheus Operator + +{{/* vim: set filetype=mustache: */}} +{{/* Expand the name of the chart. This is suffixed with -alertmanager, which means subtract 13 from longest 63 available */}} +{{- define "kube-prometheus-stack.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 50 | trimSuffix "-" -}} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +The components in this chart create additional resources that expand the longest created name strings. +The longest name that gets created adds and extra 37 characters, so truncation should be 63-35=26. +*/}} +{{- define "kube-prometheus-stack.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 26 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 26 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 26 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* Fullname suffixed with -operator */}} +{{/* Adding 9 to 26 truncation of kube-prometheus-stack.fullname */}} +{{- define "kube-prometheus-stack.operator.fullname" -}} +{{- if .Values.prometheusOperator.fullnameOverride -}} +{{- .Values.prometheusOperator.fullnameOverride | trunc 35 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-operator" (include "kube-prometheus-stack.fullname" .) -}} +{{- end }} +{{- end }} + +{{/* Prometheus custom resource instance name */}} +{{- define "kube-prometheus-stack.prometheus.crname" -}} +{{- if .Values.cleanPrometheusOperatorObjectNames }} +{{- include "kube-prometheus-stack.fullname" . }} +{{- else }} +{{- print (include "kube-prometheus-stack.fullname" .) "-prometheus" }} +{{- end }} +{{- end }} + +{{/* Prometheus apiVersion for networkpolicy */}} +{{- define "kube-prometheus-stack.prometheus.networkPolicy.apiVersion" -}} +{{- print "networking.k8s.io/v1" -}} +{{- end }} + +{{/* Alertmanager custom resource instance name */}} +{{- define "kube-prometheus-stack.alertmanager.crname" -}} +{{- if .Values.cleanPrometheusOperatorObjectNames }} +{{- include "kube-prometheus-stack.fullname" . }} +{{- else }} +{{- print (include "kube-prometheus-stack.fullname" .) "-alertmanager" -}} +{{- end }} +{{- end }} + +{{/* Fullname suffixed with thanos-ruler */}} +{{- define "kube-prometheus-stack.thanosRuler.fullname" -}} +{{- printf "%s-thanos-ruler" (include "kube-prometheus-stack.fullname" .) -}} +{{- end }} + +{{/* Shortened name suffixed with thanos-ruler */}} +{{- define "kube-prometheus-stack.thanosRuler.name" -}} +{{- default (printf "%s-thanos-ruler" (include "kube-prometheus-stack.name" .)) .Values.thanosRuler.name -}} +{{- end }} + + +{{/* Create chart name and version as used by the chart label. */}} +{{- define "kube-prometheus-stack.chartref" -}} +{{- replace "+" "_" .Chart.Version | printf "%s-%s" .Chart.Name -}} +{{- end }} + +{{/* Generate basic labels */}} +{{- define "kube-prometheus-stack.labels" }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: "{{ replace "+" "_" .Chart.Version }}" +app.kubernetes.io/part-of: {{ template "kube-prometheus-stack.name" . }} +chart: {{ template "kube-prometheus-stack.chartref" . }} +release: {{ $.Release.Name | quote }} +heritage: {{ $.Release.Service | quote }} +{{- if .Values.commonLabels}} +{{ toYaml .Values.commonLabels }} +{{- end }} +{{- end }} + +{{/* Create the name of kube-prometheus-stack service account to use */}} +{{- define "kube-prometheus-stack.operator.serviceAccountName" -}} +{{- if .Values.prometheusOperator.serviceAccount.create -}} + {{ default (include "kube-prometheus-stack.operator.fullname" .) .Values.prometheusOperator.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheusOperator.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of kube-prometheus-stack service account to use */}} +{{- define "kube-prometheus-stack.operator.admissionWebhooks.serviceAccountName" -}} +{{- if .Values.prometheusOperator.serviceAccount.create -}} + {{ default (printf "%s-webhook" (include "kube-prometheus-stack.operator.fullname" .)) .Values.prometheusOperator.admissionWebhooks.deployment.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheusOperator.admissionWebhooks.deployment.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of prometheus service account to use */}} +{{- define "kube-prometheus-stack.prometheus.serviceAccountName" -}} +{{- if .Values.prometheus.serviceAccount.create -}} + {{ default (print (include "kube-prometheus-stack.fullname" .) "-prometheus") .Values.prometheus.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheus.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of alertmanager service account to use */}} +{{- define "kube-prometheus-stack.alertmanager.serviceAccountName" -}} +{{- if .Values.alertmanager.serviceAccount.create -}} + {{ default (print (include "kube-prometheus-stack.fullname" .) "-alertmanager") .Values.alertmanager.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.alertmanager.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of thanosRuler service account to use */}} +{{- define "kube-prometheus-stack.thanosRuler.serviceAccountName" -}} +{{- if .Values.thanosRuler.serviceAccount.create -}} + {{ default (include "kube-prometheus-stack.thanosRuler.name" .) .Values.thanosRuler.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.thanosRuler.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Use the grafana namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-grafana.namespace" -}} + {{- if .Values.grafana.namespaceOverride -}} + {{- .Values.grafana.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Allow kube-state-metrics job name to be overridden +*/}} +{{- define "kube-prometheus-stack-kube-state-metrics.name" -}} + {{- if index .Values "kube-state-metrics" "nameOverride" -}} + {{- index .Values "kube-state-metrics" "nameOverride" -}} + {{- else -}} + {{- print "kube-state-metrics" -}} + {{- end -}} +{{- end -}} + +{{/* +Use the kube-state-metrics namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-kube-state-metrics.namespace" -}} + {{- if index .Values "kube-state-metrics" "namespaceOverride" -}} + {{- index .Values "kube-state-metrics" "namespaceOverride" -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Use the prometheus-node-exporter namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-prometheus-node-exporter.namespace" -}} + {{- if index .Values "prometheus-node-exporter" "namespaceOverride" -}} + {{- index .Values "prometheus-node-exporter" "namespaceOverride" -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* Allow KubeVersion to be overridden. */}} +{{- define "kube-prometheus-stack.kubeVersion" -}} + {{- default .Capabilities.KubeVersion.Version .Values.kubeVersionOverride -}} +{{- end -}} + +{{/* Get Ingress API Version */}} +{{- define "kube-prometheus-stack.ingress.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" (include "kube-prometheus-stack.kubeVersion" .)) -}} + {{- print "networking.k8s.io/v1" -}} + {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}} + {{- print "networking.k8s.io/v1beta1" -}} + {{- else -}} + {{- print "extensions/v1beta1" -}} + {{- end -}} +{{- end -}} + +{{/* Check Ingress stability */}} +{{- define "kube-prometheus-stack.ingress.isStable" -}} + {{- eq (include "kube-prometheus-stack.ingress.apiVersion" .) "networking.k8s.io/v1" -}} +{{- end -}} + +{{/* Check Ingress supports pathType */}} +{{/* pathType was added to networking.k8s.io/v1beta1 in Kubernetes 1.18 */}} +{{- define "kube-prometheus-stack.ingress.supportsPathType" -}} + {{- or (eq (include "kube-prometheus-stack.ingress.isStable" .) "true") (and (eq (include "kube-prometheus-stack.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" (include "kube-prometheus-stack.kubeVersion" .))) -}} +{{- end -}} + +{{/* Get Policy API Version */}} +{{- define "kube-prometheus-stack.pdb.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "policy/v1") (semverCompare ">= 1.21-0" (include "kube-prometheus-stack.kubeVersion" .)) -}} + {{- print "policy/v1" -}} + {{- else -}} + {{- print "policy/v1beta1" -}} + {{- end -}} + {{- end -}} + +{{/* Get value based on current Kubernetes version */}} +{{- define "kube-prometheus-stack.kubeVersionDefaultValue" -}} + {{- $values := index . 0 -}} + {{- $kubeVersion := index . 1 -}} + {{- $old := index . 2 -}} + {{- $new := index . 3 -}} + {{- $default := index . 4 -}} + {{- if kindIs "invalid" $default -}} + {{- if semverCompare $kubeVersion (include "kube-prometheus-stack.kubeVersion" $values) -}} + {{- print $new -}} + {{- else -}} + {{- print $old -}} + {{- end -}} + {{- else -}} + {{- print $default }} + {{- end -}} +{{- end -}} + +{{/* Get value for kube-controller-manager depending on insecure scraping availability */}} +{{- define "kube-prometheus-stack.kubeControllerManager.insecureScrape" -}} + {{- $values := index . 0 -}} + {{- $insecure := index . 1 -}} + {{- $secure := index . 2 -}} + {{- $userValue := index . 3 -}} + {{- include "kube-prometheus-stack.kubeVersionDefaultValue" (list $values ">= 1.22-0" $insecure $secure $userValue) -}} +{{- end -}} + +{{/* Get value for kube-scheduler depending on insecure scraping availability */}} +{{- define "kube-prometheus-stack.kubeScheduler.insecureScrape" -}} + {{- $values := index . 0 -}} + {{- $insecure := index . 1 -}} + {{- $secure := index . 2 -}} + {{- $userValue := index . 3 -}} + {{- include "kube-prometheus-stack.kubeVersionDefaultValue" (list $values ">= 1.23-0" $insecure $secure $userValue) -}} +{{- end -}} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end -}} + +{{/* +To help compatibility with other charts which use global.imagePullSecrets. +Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). +global: + imagePullSecrets: + - name: pullSecret1 + - name: pullSecret2 + +or + +global: + imagePullSecrets: + - pullSecret1 + - pullSecret2 +*/}} +{{- define "kube-prometheus-stack.imagePullSecrets" -}} +{{- range .Values.global.imagePullSecrets }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{- define "kube-prometheus-stack.operator.admission-webhook.dnsNames" }} +{{- $fullname := include "kube-prometheus-stack.operator.fullname" . }} +{{- $namespace := include "kube-prometheus-stack.namespace" . }} +{{- $fullname }} +{{ $fullname }}.{{ $namespace }}.svc +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +{{ $fullname }}-webhook +{{ $fullname }}-webhook.{{ $namespace }}.svc +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/alertmanager.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/alertmanager.yaml new file mode 100644 index 0000000000..19044054ac --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/alertmanager.yaml @@ -0,0 +1,191 @@ +{{- if .Values.alertmanager.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: Alertmanager +metadata: + name: {{ template "kube-prometheus-stack.alertmanager.crname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.annotations }} + annotations: +{{ toYaml .Values.alertmanager.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.alertmanager.alertmanagerSpec.image }} + {{- $registry := include "monitoring_registry" . | default .Values.alertmanager.alertmanagerSpec.image.registry }} + {{- if and .Values.alertmanager.alertmanagerSpec.image.tag .Values.alertmanager.alertmanagerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}:{{ .Values.alertmanager.alertmanagerSpec.image.tag }}@sha256:{{ .Values.alertmanager.alertmanagerSpec.image.sha }}" + {{- else if .Values.alertmanager.alertmanagerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}@sha256:{{ .Values.alertmanager.alertmanagerSpec.image.sha }}" + {{- else if .Values.alertmanager.alertmanagerSpec.image.tag }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}:{{ .Values.alertmanager.alertmanagerSpec.image.tag }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}" + {{- end }} + version: {{ .Values.alertmanager.alertmanagerSpec.image.tag }} + {{- if .Values.alertmanager.alertmanagerSpec.image.sha }} + sha: {{ .Values.alertmanager.alertmanagerSpec.image.sha }} + {{- end }} +{{- end }} + replicas: {{ .Values.alertmanager.alertmanagerSpec.replicas }} + listenLocal: {{ .Values.alertmanager.alertmanagerSpec.listenLocal }} + serviceAccountName: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.alertmanager.alertmanagerSpec.automountServiceAccountToken }} +{{- if .Values.alertmanager.alertmanagerSpec.externalUrl }} + externalUrl: "{{ tpl .Values.alertmanager.alertmanagerSpec.externalUrl . }}" +{{- else if and .Values.alertmanager.ingress.enabled .Values.alertmanager.ingress.hosts }} + externalUrl: "http://{{ tpl (index .Values.alertmanager.ingress.hosts 0) . }}{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" +{{- else if not (or (kindIs "invalid" .Values.global.cattle.url) (kindIs "invalid" .Values.global.cattle.clusterId)) }} + externalUrl: "{{ .Values.global.cattle.url }}/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ .Values.namespaceOverride }}/services/http:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}/proxy" +{{- else }} + externalUrl: http://{{ template "kube-prometheus-stack.fullname" . }}-alertmanager.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.alertmanager.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.alertmanager.alertmanagerSpec.nodeSelector }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.alertmanager.alertmanagerSpec.paused }} + logFormat: {{ .Values.alertmanager.alertmanagerSpec.logFormat | quote }} + logLevel: {{ .Values.alertmanager.alertmanagerSpec.logLevel | quote }} + retention: {{ .Values.alertmanager.alertmanagerSpec.retention | quote }} +{{- if .Values.alertmanager.alertmanagerSpec.secrets }} + secrets: +{{ toYaml .Values.alertmanager.alertmanagerSpec.secrets | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.configSecret }} + configSecret: {{ .Values.alertmanager.alertmanagerSpec.configSecret }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.configMaps }} + configMaps: +{{ toYaml .Values.alertmanager.alertmanagerSpec.configMaps | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector }} + alertmanagerConfigSelector: +{{ tpl (toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector | indent 4) . }} +{{ else }} + alertmanagerConfigSelector: {} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigNamespaceSelector }} + alertmanagerConfigNamespaceSelector: +{{ tpl (toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigNamespaceSelector | indent 4) . }} +{{ else }} + alertmanagerConfigNamespaceSelector: {} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.web }} + web: +{{ toYaml .Values.alertmanager.alertmanagerSpec.web | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfiguration }} + alertmanagerConfiguration: +{{ toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfiguration | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy }} + alertmanagerConfigMatcherStrategy: +{{ toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.resources }} + resources: +{{ toYaml .Values.alertmanager.alertmanagerSpec.resources | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.routePrefix }} + routePrefix: "{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.securityContext }} + securityContext: +{{ toYaml .Values.alertmanager.alertmanagerSpec.securityContext | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.storage }} + storage: +{{ tpl (toYaml .Values.alertmanager.alertmanagerSpec.storage | indent 4) . }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.podMetadata }} + podMetadata: +{{ toYaml .Values.alertmanager.alertmanagerSpec.podMetadata | indent 4 }} +{{- end }} +{{- if or .Values.alertmanager.alertmanagerSpec.podAntiAffinity .Values.alertmanager.alertmanagerSpec.affinity }} + affinity: +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.affinity }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.alertmanager.alertmanagerSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.alertmanager.alertmanagerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [alertmanager]} + - {key: alertmanager, operator: In, values: [{{ template "kube-prometheus-stack.alertmanager.crname" . }}]} +{{- else if eq .Values.alertmanager.alertmanagerSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.alertmanager.alertmanagerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [alertmanager]} + - {key: alertmanager, operator: In, values: [{{ template "kube-prometheus-stack.alertmanager.crname" . }}]} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.alertmanager.alertmanagerSpec.tolerations }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.alertmanager.alertmanagerSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.containers }} + containers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.containers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.initContainers }} + initContainers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.priorityClassName }} + priorityClassName: {{.Values.alertmanager.alertmanagerSpec.priorityClassName }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.additionalPeers }} + additionalPeers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.additionalPeers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.volumes }} + volumes: +{{ toYaml .Values.alertmanager.alertmanagerSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.alertmanager.alertmanagerSpec.volumeMounts | indent 4 }} +{{- end }} + portName: {{ .Values.alertmanager.alertmanagerSpec.portName }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterAdvertiseAddress }} + clusterAdvertiseAddress: {{ .Values.alertmanager.alertmanagerSpec.clusterAdvertiseAddress }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterGossipInterval }} + clusterGossipInterval: {{ .Values.alertmanager.alertmanagerSpec.clusterGossipInterval }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterPeerTimeout }} + clusterPeerTimeout: {{ .Values.alertmanager.alertmanagerSpec.clusterPeerTimeout }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterPushpullInterval }} + clusterPushpullInterval: {{ .Values.alertmanager.alertmanagerSpec.clusterPushpullInterval }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.forceEnableClusterMode }} + forceEnableClusterMode: {{ .Values.alertmanager.alertmanagerSpec.forceEnableClusterMode }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.minReadySeconds }} + minReadySeconds: {{ .Values.alertmanager.alertmanagerSpec.minReadySeconds }} +{{- end }} +{{- with .Values.alertmanager.alertmanagerSpec.additionalConfig }} + {{- tpl (toYaml .) $ | nindent 2 }} +{{- end }} +{{- with .Values.alertmanager.alertmanagerSpec.additionalConfigString }} + {{- tpl . $ | nindent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/extrasecret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/extrasecret.yaml new file mode 100644 index 0000000000..ecd8f47021 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.alertmanager.extraSecret.data -}} +{{- $secretName := printf "alertmanager-%s-extra" (include "kube-prometheus-stack.fullname" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.alertmanager.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.alertmanager.extraSecret.annotations }} + annotations: +{{ toYaml .Values.alertmanager.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/component: alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.alertmanager.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/ingress.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/ingress.yaml new file mode 100644 index 0000000000..be9f5aa279 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/ingress.yaml @@ -0,0 +1,78 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.ingress.enabled }} +{{- $pathType := .Values.alertmanager.ingress.pathType | default "ImplementationSpecific" }} +{{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager" }} +{{- $backendServiceName := .Values.alertmanager.ingress.serviceName | default (printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager") }} +{{- $servicePort := .Values.alertmanager.ingress.servicePort | default .Values.alertmanager.service.port -}} +{{- $routePrefix := list .Values.alertmanager.alertmanagerSpec.routePrefix }} +{{- $paths := .Values.alertmanager.ingress.paths | default $routePrefix -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.alertmanager.ingress.annotations }} + annotations: + {{- tpl (toYaml .Values.alertmanager.ingress.annotations) . | nindent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{- if .Values.alertmanager.ingress.labels }} +{{ toYaml .Values.alertmanager.ingress.labels | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if $apiIsStable }} + {{- if .Values.alertmanager.ingress.ingressClassName }} + ingressClassName: {{ .Values.alertmanager.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.alertmanager.ingress.hosts }} + {{- range $host := .Values.alertmanager.ingress.hosts }} + - host: {{ tpl $host $ | quote }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $backendServiceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $backendServiceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $backendServiceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $backendServiceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.alertmanager.ingress.tls }} + tls: +{{ tpl (toYaml .Values.alertmanager.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/ingressperreplica.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/ingressperreplica.yaml new file mode 100644 index 0000000000..b2e00a4162 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/ingressperreplica.yaml @@ -0,0 +1,67 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.servicePerReplica.enabled .Values.alertmanager.ingressPerReplica.enabled }} +{{- $pathType := .Values.alertmanager.ingressPerReplica.pathType | default "" }} +{{- $count := .Values.alertmanager.alertmanagerSpec.replicas | int -}} +{{- $servicePort := .Values.alertmanager.service.port -}} +{{- $ingressValues := .Values.alertmanager.ingressPerReplica -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-ingressperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{ range $i, $e := until $count }} + - kind: Ingress + apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" $ }} + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-alertmanager + {{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $ingressValues.labels }} +{{ toYaml $ingressValues.labels | indent 8 }} + {{- end }} + {{- if $ingressValues.annotations }} + annotations: + {{- tpl (toYaml $ingressValues.annotations) $ | nindent 8 }} + {{- end }} + spec: + {{- if $apiIsStable }} + {{- if $ingressValues.ingressClassName }} + ingressClassName: {{ $ingressValues.ingressClassName }} + {{- end }} + {{- end }} + rules: + - host: {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + http: + paths: + {{- range $p := $ingressValues.paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- if or $ingressValues.tlsSecretName $ingressValues.tlsSecretPerReplica.enabled }} + tls: + - hosts: + - {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + {{- if $ingressValues.tlsSecretPerReplica.enabled }} + secretName: {{ $ingressValues.tlsSecretPerReplica.prefix }}-{{ $i }} + {{- else }} + secretName: {{ $ingressValues.tlsSecretName }} + {{- end }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/podDisruptionBudget.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/podDisruptionBudget.yaml new file mode 100644 index 0000000000..b183403125 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/podDisruptionBudget.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.podDisruptionBudget.enabled }} +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.alertmanager.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.alertmanager.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.alertmanager.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.alertmanager.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "kube-prometheus-stack.alertmanager.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/psp-role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/psp-role.yaml new file mode 100644 index 0000000000..8810e93ded --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/psp-role.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.alertmanager.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/psp-rolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/psp-rolebinding.yaml new file mode 100644 index 0000000000..794f4ad178 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/psp-rolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.alertmanager.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/psp.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/psp.yaml new file mode 100644 index 0000000000..07b616b5cb --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/psp.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.alertmanager.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/secret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/secret.yaml new file mode 100644 index 0000000000..d2fe84a7bf --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/secret.yaml @@ -0,0 +1,35 @@ +{{- if and (.Values.alertmanager.enabled) (not .Values.alertmanager.alertmanagerSpec.useExistingSecret) }} +{{/* This file is applied when the operation is helm install and the target secret does not exist. */}} +{{- $secretName := (printf "alertmanager-%s" (include "kube-prometheus-stack.alertmanager.crname" .)) }} +{{- if or (not (lookup "v1" "Secret" (include "kube-prometheus-stack.namespace" .) $secretName)) (eq .Values.alertmanager.secret.recreateIfExists true) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install, pre-upgrade + "helm.sh/hook-weight": "3" + "helm.sh/resource-policy": keep +{{- if .Values.alertmanager.secret.annotations }} +{{ toYaml .Values.alertmanager.secret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- if .Values.alertmanager.tplConfig }} +{{- if .Values.alertmanager.stringConfig }} + alertmanager.yaml: {{ tpl (.Values.alertmanager.stringConfig) . | b64enc | quote }} +{{- else if eq (typeOf .Values.alertmanager.config) "string" }} + alertmanager.yaml: {{ tpl (.Values.alertmanager.config) . | b64enc | quote }} +{{- else }} + alertmanager.yaml: {{ tpl (toYaml .Values.alertmanager.config) . | b64enc | quote }} +{{- end }} +{{- else }} + alertmanager.yaml: {{ toYaml .Values.alertmanager.config | b64enc | quote }} +{{- end }} +{{- range $key, $val := .Values.alertmanager.templateFiles }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/service.yaml new file mode 100644 index 0000000000..373de328a5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/service.yaml @@ -0,0 +1,68 @@ +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if .Values.alertmanager.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + self-monitor: {{ .Values.alertmanager.serviceMonitor.selfMonitor | quote }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.service.labels }} +{{ toYaml .Values.alertmanager.service.labels | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.service.annotations }} + annotations: +{{ toYaml .Values.alertmanager.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.alertmanager.service.clusterIP }} + clusterIP: {{ .Values.alertmanager.service.clusterIP }} +{{- end }} +{{- if .Values.alertmanager.service.externalIPs }} + externalIPs: +{{ toYaml .Values.alertmanager.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.alertmanager.service.loadBalancerIP }} +{{- end }} +{{- if .Values.alertmanager.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.alertmanager.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.alertmanager.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.alertmanager.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.alertmanager.alertmanagerSpec.portName }} + {{- if eq .Values.alertmanager.service.type "NodePort" }} + nodePort: {{ .Values.alertmanager.service.nodePort }} + {{- end }} + port: {{ .Values.alertmanager.service.port }} + targetPort: {{ .Values.alertmanager.service.targetPort }} + protocol: TCP + - name: reloader-web + {{- if semverCompare ">=1.20.0-0" $kubeTargetVersion }} + appProtocol: http + {{- end }} + port: 8080 + targetPort: reloader-web +{{- if .Values.alertmanager.service.additionalPorts }} +{{ toYaml .Values.alertmanager.service.additionalPorts | indent 2 }} +{{- end }} + selector: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "kube-prometheus-stack.alertmanager.crname" . }} +{{- if .Values.alertmanager.service.sessionAffinity }} + sessionAffinity: {{ .Values.alertmanager.service.sessionAffinity }} +{{- end }} +{{- if eq .Values.alertmanager.service.sessionAffinity "ClientIP" }} + sessionAffinityConfig: + clientIP: + timeoutSeconds: {{ .Values.alertmanager.service.sessionAffinityConfig.clientIP.timeoutSeconds }} +{{- end }} + type: "{{ .Values.alertmanager.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/serviceaccount.yaml new file mode 100644 index 0000000000..745ced8bde --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/serviceaccount.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/component: alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.alertmanager.serviceAccount.annotations | indent 4 }} +{{- end }} +automountServiceAccountToken: {{ .Values.alertmanager.serviceAccount.automountServiceAccountToken }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2}} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/servicemonitor.yaml new file mode 100644 index 0000000000..6233690019 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/servicemonitor.yaml @@ -0,0 +1,84 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.alertmanager.serviceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.alertmanager.serviceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + release: {{ $.Release.Name | quote }} + self-monitor: "true" + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.alertmanager.alertmanagerSpec.portName }} + enableHttp2: {{ .Values.alertmanager.serviceMonitor.enableHttp2 }} + {{- if .Values.alertmanager.serviceMonitor.interval }} + interval: {{ .Values.alertmanager.serviceMonitor.interval }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.alertmanager.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.scheme }} + scheme: {{ .Values.alertmanager.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.alertmanager.serviceMonitor.bearerTokenFile }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.alertmanager.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + path: "{{ trimSuffix "/" .Values.alertmanager.alertmanagerSpec.routePrefix }}/metrics" + metricRelabelings: + {{- if .Values.alertmanager.serviceMonitor.metricRelabelings }} + {{- tpl (toYaml .Values.alertmanager.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.alertmanager.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + {{- range .Values.alertmanager.serviceMonitor.additionalEndpoints }} + - port: {{ .port }} + {{- if or $.Values.alertmanager.serviceMonitor.interval .interval }} + interval: {{ default $.Values.alertmanager.serviceMonitor.interval .interval }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.proxyUrl .proxyUrl }} + proxyUrl: {{ default $.Values.alertmanager.serviceMonitor.proxyUrl .proxyUrl }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.scheme .scheme }} + scheme: {{ default $.Values.alertmanager.serviceMonitor.scheme .scheme }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.bearerTokenFile .bearerTokenFile }} + bearerTokenFile: {{ default $.Values.alertmanager.serviceMonitor.bearerTokenFile .bearerTokenFile }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.tlsConfig .tlsConfig }} + tlsConfig: {{- default $.Values.alertmanager.serviceMonitor.tlsConfig .tlsConfig | toYaml | nindent 6 }} + {{- end }} + path: {{ .path }} + {{- if or $.Values.alertmanager.serviceMonitor.metricRelabelings .metricRelabelings }} + metricRelabelings: {{- tpl (default $.Values.alertmanager.serviceMonitor.metricRelabelings .metricRelabelings | toYaml | nindent 6) . }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.relabelings .relabelings }} + relabelings: {{- default $.Values.alertmanager.serviceMonitor.relabelings .relabelings | toYaml | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/serviceperreplica.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/serviceperreplica.yaml new file mode 100644 index 0000000000..75a13bdf97 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/alertmanager/serviceperreplica.yaml @@ -0,0 +1,49 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.servicePerReplica.enabled }} +{{- $count := .Values.alertmanager.alertmanagerSpec.replicas | int -}} +{{- $serviceValues := .Values.alertmanager.servicePerReplica -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-serviceperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- range $i, $e := until $count }} + - apiVersion: v1 + kind: Service + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-alertmanager +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $serviceValues.annotations }} + annotations: +{{ toYaml $serviceValues.annotations | indent 8 }} + {{- end }} + spec: + {{- if $serviceValues.clusterIP }} + clusterIP: {{ $serviceValues.clusterIP }} + {{- end }} + {{- if $serviceValues.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := $serviceValues.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} + {{- end }} + {{- if ne $serviceValues.type "ClusterIP" }} + externalTrafficPolicy: {{ $serviceValues.externalTrafficPolicy }} + {{- end }} + ports: + - name: {{ $.Values.alertmanager.alertmanagerSpec.portName }} + {{- if eq $serviceValues.type "NodePort" }} + nodePort: {{ $serviceValues.nodePort }} + {{- end }} + port: {{ $serviceValues.port }} + targetPort: {{ $serviceValues.targetPort }} + selector: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "kube-prometheus-stack.alertmanager.crname" $ }} + statefulset.kubernetes.io/pod-name: alertmanager-{{ include "kube-prometheus-stack.alertmanager.crname" $ }}-{{ $i }} + type: "{{ $serviceValues.type }}" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/core-dns/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/core-dns/service.yaml new file mode 100644 index 0000000000..b8618f7558 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/core-dns/service.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.coreDns.enabled .Values.coreDns.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-coredns + labels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + jobLabel: coredns +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.coreDns.serviceMonitor.port }} + port: {{ .Values.coreDns.service.port }} + protocol: TCP + targetPort: {{ .Values.coreDns.service.targetPort }} + selector: + {{- if .Values.coreDns.service.selector }} +{{ toYaml .Values.coreDns.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-dns + {{- end}} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/core-dns/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/core-dns/servicemonitor.yaml new file mode 100644 index 0000000000..dc15a06937 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/core-dns/servicemonitor.yaml @@ -0,0 +1,58 @@ +{{- if and .Values.coreDns.enabled .Values.coreDns.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-coredns + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + {{- with .Values.coreDns.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.coreDns.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.coreDns.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.coreDns.serviceMonitor.selector }} + {{ tpl (toYaml .Values.coreDns.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.coreDns.serviceMonitor.port }} + {{- if .Values.coreDns.serviceMonitor.interval}} + interval: {{ .Values.coreDns.serviceMonitor.interval }} + {{- end }} + {{- if .Values.coreDns.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.coreDns.serviceMonitor.proxyUrl}} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + metricRelabelings: + {{- if .Values.coreDns.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.coreDns.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.coreDns.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.coreDns.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-api-server/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-api-server/servicemonitor.yaml new file mode 100644 index 0000000000..66e777632e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-api-server/servicemonitor.yaml @@ -0,0 +1,57 @@ +{{- if and .Values.kubeApiServer.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-apiserver + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: default + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-apiserver + {{- with .Values.kubeApiServer.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.kubeApiServer.serviceMonitor | nindent 2 }} + endpoints: + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeApiServer.serviceMonitor.interval }} + interval: {{ .Values.kubeApiServer.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubeApiServer.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeApiServer.serviceMonitor.proxyUrl }} + {{- end }} + port: https + scheme: https + metricRelabelings: + {{- if .Values.kubeApiServer.serviceMonitor.metricRelabelings }} +{{ tpl (toYaml .Values.kubeApiServer.serviceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeApiServer.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeApiServer.serviceMonitor.relabelings | indent 6) . }} +{{- end }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + serverName: {{ .Values.kubeApiServer.tlsConfig.serverName }} + insecureSkipVerify: {{ .Values.kubeApiServer.tlsConfig.insecureSkipVerify }} + jobLabel: {{ .Values.kubeApiServer.serviceMonitor.jobLabel }} + namespaceSelector: + matchNames: + - default + selector: +{{ toYaml .Values.kubeApiServer.serviceMonitor.selector | indent 4 }} +{{- end}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-controller-manager/endpoints.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-controller-manager/endpoints.yaml new file mode 100644 index 0000000000..6a6afa6412 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-controller-manager/endpoints.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.kubeControllerManager.enabled .Values.kubeControllerManager.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + k8s-app: kube-controller-manager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeControllerManager.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeControllerManager.serviceMonitor.port }} + {{- $kubeControllerManagerDefaultInsecurePort := 10252 }} + {{- $kubeControllerManagerDefaultSecurePort := 10257 }} + port: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.port) }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-controller-manager/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-controller-manager/service.yaml new file mode 100644 index 0000000000..43b1a976d5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-controller-manager/service.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.kubeControllerManager.enabled .Values.kubeControllerManager.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + jobLabel: kube-controller-manager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.kubeControllerManager.serviceMonitor.port }} + {{- $kubeControllerManagerDefaultInsecurePort := 10252 }} + {{- $kubeControllerManagerDefaultSecurePort := 10257 }} + port: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.port) }} + protocol: TCP + targetPort: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.targetPort) }} +{{- if .Values.kubeControllerManager.endpoints }}{{- else }} + selector: + {{- if .Values.kubeControllerManager.service.selector }} +{{ toYaml .Values.kubeControllerManager.service.selector | indent 4 }} + {{- else}} + component: kube-controller-manager + {{- end}} +{{- end }} + type: ClusterIP +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-controller-manager/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-controller-manager/servicemonitor.yaml new file mode 100644 index 0000000000..7ed3baa65f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-controller-manager/servicemonitor.yaml @@ -0,0 +1,69 @@ +{{- if and .Values.kubeControllerManager.enabled .Values.kubeControllerManager.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + {{- with .Values.kubeControllerManager.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeControllerManager.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeControllerManager.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeControllerManager.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeControllerManager.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeControllerManager.serviceMonitor.port }} + {{- if .Values.kubeControllerManager.serviceMonitor.interval }} + interval: {{ .Values.kubeControllerManager.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeControllerManager.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeControllerManager.serviceMonitor.proxyUrl}} + {{- end }} + {{- if eq (include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . false true .Values.kubeControllerManager.serviceMonitor.https )) "true" }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- if eq (include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . nil true .Values.kubeControllerManager.serviceMonitor.insecureSkipVerify)) "true" }} + insecureSkipVerify: true + {{- end }} + {{- if .Values.kubeControllerManager.serviceMonitor.serverName }} + serverName: {{ .Values.kubeControllerManager.serviceMonitor.serverName }} + {{- end }} + {{- end }} + metricRelabelings: + {{- if.Values.kubeControllerManager.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeControllerManager.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeControllerManager.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeControllerManager.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-dns/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-dns/service.yaml new file mode 100644 index 0000000000..81b2c9930c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-dns/service.yaml @@ -0,0 +1,28 @@ +{{- if and .Values.kubeDns.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-dns + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + jobLabel: kube-dns +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: http-metrics-dnsmasq + port: {{ .Values.kubeDns.service.dnsmasq.port }} + protocol: TCP + targetPort: {{ .Values.kubeDns.service.dnsmasq.targetPort }} + - name: http-metrics-skydns + port: {{ .Values.kubeDns.service.skydns.port }} + protocol: TCP + targetPort: {{ .Values.kubeDns.service.skydns.targetPort }} + selector: + {{- if .Values.kubeDns.service.selector }} +{{ toYaml .Values.kubeDns.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-dns + {{- end}} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-dns/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-dns/servicemonitor.yaml new file mode 100644 index 0000000000..9fa41b575f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-dns/servicemonitor.yaml @@ -0,0 +1,71 @@ +{{- if and .Values.kubeDns.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-dns + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + {{- with .Values.kubeDns.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeDns.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeDns.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeDns.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeDns.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: http-metrics-dnsmasq + {{- if .Values.kubeDns.serviceMonitor.interval }} + interval: {{ .Values.kubeDns.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeDns.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeDns.serviceMonitor.proxyUrl}} + {{- end }} + metricRelabelings: + {{- if .Values.kubeDns.serviceMonitor.dnsmasqMetricRelabelings }} + {{ tpl (toYaml .Values.kubeDns.serviceMonitor.dnsmasqMetricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeDns.serviceMonitor.dnsmasqRelabelings }} + relabelings: +{{ toYaml .Values.kubeDns.serviceMonitor.dnsmasqRelabelings | indent 4 }} +{{- end }} + - port: http-metrics-skydns + {{- if .Values.kubeDns.serviceMonitor.interval }} + interval: {{ .Values.kubeDns.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubeDns.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubeDns.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubeDns.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeDns.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-etcd/endpoints.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-etcd/endpoints.yaml new file mode 100644 index 0000000000..e366447577 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-etcd/endpoints.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.kubeEtcd.enabled .Values.kubeEtcd.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + k8s-app: etcd-server +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeEtcd.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeEtcd.serviceMonitor.port }} + port: {{ .Values.kubeEtcd.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-etcd/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-etcd/service.yaml new file mode 100644 index 0000000000..d07d4f35e3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-etcd/service.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.kubeEtcd.enabled .Values.kubeEtcd.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + jobLabel: kube-etcd +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.kubeEtcd.serviceMonitor.port }} + port: {{ .Values.kubeEtcd.service.port }} + protocol: TCP + targetPort: {{ .Values.kubeEtcd.service.targetPort }} +{{- if .Values.kubeEtcd.endpoints }}{{- else }} + selector: + {{- if .Values.kubeEtcd.service.selector }} +{{ toYaml .Values.kubeEtcd.service.selector | indent 4 }} + {{- else}} + component: etcd + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-etcd/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-etcd/servicemonitor.yaml new file mode 100644 index 0000000000..26fdbdbed3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-etcd/servicemonitor.yaml @@ -0,0 +1,75 @@ +{{- if and .Values.kubeEtcd.enabled .Values.kubeEtcd.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + {{- with .Values.kubeEtcd.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeEtcd.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeEtcd.serviceMonitor | nindent 4 }} + selector: + {{- if .Values.kubeEtcd.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeEtcd.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeEtcd.serviceMonitor.port }} + {{- if .Values.kubeEtcd.serviceMonitor.interval }} + interval: {{ .Values.kubeEtcd.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeEtcd.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeEtcd.serviceMonitor.proxyUrl}} + {{- end }} + {{- if eq .Values.kubeEtcd.serviceMonitor.scheme "https" }} + scheme: https + tlsConfig: + {{- if .Values.kubeEtcd.serviceMonitor.serverName }} + serverName: {{ .Values.kubeEtcd.serviceMonitor.serverName }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.caFile }} + caFile: {{ .Values.kubeEtcd.serviceMonitor.caFile }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.certFile }} + certFile: {{ .Values.kubeEtcd.serviceMonitor.certFile }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.keyFile }} + keyFile: {{ .Values.kubeEtcd.serviceMonitor.keyFile }} + {{- end}} + insecureSkipVerify: {{ .Values.kubeEtcd.serviceMonitor.insecureSkipVerify }} + {{- end }} + metricRelabelings: + {{- if .Values.kubeEtcd.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeEtcd.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeEtcd.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeEtcd.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-proxy/endpoints.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-proxy/endpoints.yaml new file mode 100644 index 0000000000..8613e62425 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-proxy/endpoints.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.kubeProxy.enabled .Values.kubeProxy.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + k8s-app: kube-proxy +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeProxy.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeProxy.serviceMonitor.port }} + port: {{ .Values.kubeProxy.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-proxy/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-proxy/service.yaml new file mode 100644 index 0000000000..8ccb2210d7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-proxy/service.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.kubeProxy.enabled .Values.kubeProxy.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + jobLabel: kube-proxy +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.kubeProxy.serviceMonitor.port }} + port: {{ .Values.kubeProxy.service.port }} + protocol: TCP + targetPort: {{ .Values.kubeProxy.service.targetPort }} +{{- if .Values.kubeProxy.endpoints }}{{- else }} + selector: + {{- if .Values.kubeProxy.service.selector }} +{{ toYaml .Values.kubeProxy.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-proxy + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-proxy/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-proxy/servicemonitor.yaml new file mode 100644 index 0000000000..24b0ab2001 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-proxy/servicemonitor.yaml @@ -0,0 +1,63 @@ +{{- if and .Values.kubeProxy.enabled .Values.kubeProxy.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + {{- with .Values.kubeProxy.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeProxy.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeProxy.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeProxy.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeProxy.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeProxy.serviceMonitor.port }} + {{- if .Values.kubeProxy.serviceMonitor.interval }} + interval: {{ .Values.kubeProxy.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeProxy.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeProxy.serviceMonitor.proxyUrl}} + {{- end }} + {{- if .Values.kubeProxy.serviceMonitor.https }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- end}} + metricRelabelings: + {{- if .Values.kubeProxy.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeProxy.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeProxy.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeProxy.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-scheduler/endpoints.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-scheduler/endpoints.yaml new file mode 100644 index 0000000000..6236b42f10 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-scheduler/endpoints.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.kubeScheduler.enabled .Values.kubeScheduler.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + k8s-app: kube-scheduler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeScheduler.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeScheduler.serviceMonitor.port }} + {{- $kubeSchedulerDefaultInsecurePort := 10251 }} + {{- $kubeSchedulerDefaultSecurePort := 10259 }} + port: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.port) }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-scheduler/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-scheduler/service.yaml new file mode 100644 index 0000000000..90b3a800a4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-scheduler/service.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.kubeScheduler.enabled .Values.kubeScheduler.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + jobLabel: kube-scheduler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.kubeScheduler.serviceMonitor.port }} + {{- $kubeSchedulerDefaultInsecurePort := 10251 }} + {{- $kubeSchedulerDefaultSecurePort := 10259 }} + port: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.port) }} + protocol: TCP + targetPort: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.targetPort) }} +{{- if .Values.kubeScheduler.endpoints }}{{- else }} + selector: + {{- if .Values.kubeScheduler.service.selector }} +{{ toYaml .Values.kubeScheduler.service.selector | indent 4 }} + {{- else}} + component: kube-scheduler + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-scheduler/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-scheduler/servicemonitor.yaml new file mode 100644 index 0000000000..b17c4f1d47 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-scheduler/servicemonitor.yaml @@ -0,0 +1,69 @@ +{{- if and .Values.kubeScheduler.enabled .Values.kubeScheduler.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + {{- with .Values.kubeScheduler.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeScheduler.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeScheduler.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeScheduler.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeScheduler.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeScheduler.serviceMonitor.port }} + {{- if .Values.kubeScheduler.serviceMonitor.interval }} + interval: {{ .Values.kubeScheduler.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeScheduler.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeScheduler.serviceMonitor.proxyUrl}} + {{- end }} + {{- if eq (include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . false true .Values.kubeScheduler.serviceMonitor.https )) "true" }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- if eq (include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . nil true .Values.kubeScheduler.serviceMonitor.insecureSkipVerify)) "true" }} + insecureSkipVerify: true + {{- end }} + {{- if .Values.kubeScheduler.serviceMonitor.serverName }} + serverName: {{ .Values.kubeScheduler.serviceMonitor.serverName }} + {{- end}} + {{- end}} + metricRelabelings: + {{- if .Values.kubeScheduler.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeScheduler.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeScheduler.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeScheduler.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-state-metrics/validate.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-state-metrics/validate.yaml new file mode 100644 index 0000000000..9211b3d771 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kube-state-metrics/validate.yaml @@ -0,0 +1,7 @@ +{{- if .Values.kubeStateMetrics.enabled }} +{{- if not (kindIs "invalid" .Values.kubeStateMetrics.serviceMonitor) }} +{{- if .Values.kubeStateMetrics.serviceMonitor.namespaceOverride }} +{{- fail "kubeStateMetrics.serviceMonitor.namespaceOverride was removed. Please use kube-state-metrics.namespaceOverride instead." }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kubelet/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kubelet/servicemonitor.yaml new file mode 100644 index 0000000000..f570fbfdbc --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/kubelet/servicemonitor.yaml @@ -0,0 +1,246 @@ +{{- if (and (not .Values.kubelet.enabled) .Values.hardenedKubelet.enabled) }} +{{ required "Cannot set .Values.hardenedKubelet.enabled=true when .Values.kubelet.enabled=false" "" }} +{{- end }} +{{- if (and .Values.kubelet.enabled .Values.kubernetesServiceMonitors.enabled (not .Values.hardenedKubelet.enabled) (not .Values.k3sServer.enabled)) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kubelet + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: {{ .Values.kubelet.namespace }} + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kubelet + {{- with .Values.kubelet.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.kubelet.serviceMonitor | nindent 2 }} + {{- with .Values.kubelet.serviceMonitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + endpoints: + {{- if .Values.kubelet.serviceMonitor.https }} + - port: https-metrics + scheme: https + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + metricRelabelings: + {{- if .Values.kubelet.serviceMonitor.metricRelabelings }} + {{- tpl (toYaml .Values.kubelet.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubelet.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisor }} + - port: https-metrics + scheme: https + path: /metrics/cadvisor + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisorRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probes }} + - port: https-metrics + scheme: https + path: /metrics/probes + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.probesMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probesRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resource }} + - port: https-metrics + scheme: https + path: {{ include "kubelet.serviceMonitor.resourcePath" . }} + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.resourceMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resourceRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceRelabelings | indent 4) . }} +{{- end }} +{{- end }} + {{- else }} + - port: http-metrics + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisor }} + - port: http-metrics + path: /metrics/cadvisor + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisorRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probes }} + - port: http-metrics + path: /metrics/probes + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.probesMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probesRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resource }} + - port: http-metrics + path: {{ include "kubelet.serviceMonitor.resourcePath" . }} + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.resourceMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resourceRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- end }} + {{- end }} + jobLabel: k8s-app + namespaceSelector: + matchNames: + - {{ .Values.kubelet.namespace }} + selector: + matchLabels: + app.kubernetes.io/name: kubelet + k8s-app: kubelet +{{- end}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/node-exporter/validate.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/node-exporter/validate.yaml new file mode 100644 index 0000000000..bdc73d6165 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/exporters/node-exporter/validate.yaml @@ -0,0 +1,3 @@ +{{- if (and (not .Values.nodeExporter.enabled) .Values.hardenedNodeExporter.enabled) }} +{{ required "Cannot set .Values.hardenedNodeExporter.enabled=true when .Values.nodeExporter.enabled=false" "" }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/extra-objects.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/extra-objects.yaml new file mode 100644 index 0000000000..567f7bf329 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/extra-objects.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraManifests }} +--- +{{ tpl (toYaml .) $ }} +{{ end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/configmap-dashboards.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/configmap-dashboards.yaml new file mode 100644 index 0000000000..e719009ffe --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/configmap-dashboards.yaml @@ -0,0 +1,24 @@ +{{- if or (and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled) .Values.grafana.forceDeployDashboards }} +{{- $files := .Files.Glob "dashboards-1.14/*.json" }} +{{- if $files }} +apiVersion: v1 +kind: ConfigMapList +items: +{{- range $path, $fileContents := $files }} +{{- $dashboardName := regexReplaceAll "(^.*/)(.*)\\.json$" $path "${2}" }} +- apiVersion: v1 + kind: ConfigMap + metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) $dashboardName | trunc 63 | trimSuffix "-" }} + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 6 }} + data: + {{ $dashboardName }}.json: {{ $.Files.Get $path | toJson }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/configmaps-datasources.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/configmaps-datasources.yaml new file mode 100644 index 0000000000..718020d4f6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/configmaps-datasources.yaml @@ -0,0 +1,81 @@ +{{- if or (and .Values.grafana.enabled .Values.grafana.sidecar.datasources.enabled) .Values.grafana.forceDeployDatasources }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-grafana-datasource + namespace: {{ default .Values.grafana.sidecar.datasources.searchNamespace (include "kube-prometheus-stack.namespace" .) }} +{{- if .Values.grafana.sidecar.datasources.annotations }} + annotations: + {{- toYaml .Values.grafana.sidecar.datasources.annotations | nindent 4 }} +{{- end }} + labels: + {{ $.Values.grafana.sidecar.datasources.label }}: {{ $.Values.grafana.sidecar.datasources.labelValue | quote }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + datasource.yaml: |- + apiVersion: 1 +{{- if .Values.grafana.deleteDatasources }} + deleteDatasources: +{{ tpl (toYaml .Values.grafana.deleteDatasources | indent 6) . }} +{{- end }} + datasources: +{{- $scrapeInterval := .Values.grafana.sidecar.datasources.defaultDatasourceScrapeInterval | default .Values.prometheus.prometheusSpec.scrapeInterval | default "30s" }} +{{- if .Values.grafana.sidecar.datasources.defaultDatasourceEnabled }} + - name: Prometheus + type: prometheus + uid: {{ .Values.grafana.sidecar.datasources.uid }} + {{- if .Values.grafana.sidecar.datasources.url }} + url: {{ .Values.grafana.sidecar.datasources.url }} + {{- else }} + url: http://{{ template "kube-prometheus-stack.fullname" . }}-prometheus.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.prometheus.service.port }}/{{ trimPrefix "/" .Values.prometheus.prometheusSpec.routePrefix }} + {{- end }} + access: proxy + isDefault: {{ .Values.grafana.sidecar.datasources.isDefaultDatasource }} + jsonData: + httpMethod: {{ .Values.grafana.sidecar.datasources.httpMethod }} + timeInterval: {{ $scrapeInterval }} + {{- if .Values.grafana.sidecar.datasources.timeout }} + timeout: {{ .Values.grafana.sidecar.datasources.timeout }} + {{- end }} +{{- if .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations }} + exemplarTraceIdDestinations: + - datasourceUid: {{ .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.datasourceUid }} + name: {{ .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.traceIdLabelName }} +{{- end }} +{{- if .Values.grafana.sidecar.datasources.createPrometheusReplicasDatasources }} +{{- range until (int .Values.prometheus.prometheusSpec.replicas) }} + - name: Prometheus-{{ . }} + type: prometheus + uid: {{ $.Values.grafana.sidecar.datasources.uid }}-replica-{{ . }} + url: http://prometheus-{{ template "kube-prometheus-stack.prometheus.crname" $ }}-{{ . }}.prometheus-operated:9090/{{ trimPrefix "/" $.Values.prometheus.prometheusSpec.routePrefix }} + access: proxy + isDefault: false + jsonData: + timeInterval: {{ $scrapeInterval }} +{{- if $.Values.grafana.sidecar.datasources.exemplarTraceIdDestinations }} + exemplarTraceIdDestinations: + - datasourceUid: {{ $.Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.datasourceUid }} + name: {{ $.Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.traceIdLabelName }} +{{- end }} +{{- end }} +{{- end }} +{{- if .Values.grafana.sidecar.datasources.alertmanager.enabled }} + - name: Alertmanager + type: alertmanager + uid: {{ .Values.grafana.sidecar.datasources.alertmanager.uid }} + {{- if .Values.grafana.sidecar.datasources.alertmanager.url }} + url: {{ .Values.grafana.sidecar.datasources.alertmanager.url }} + {{- else }} + url: http://{{ template "kube-prometheus-stack.fullname" . }}-alertmanager.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.alertmanager.service.port }}/{{ trimPrefix "/" .Values.alertmanager.alertmanagerSpec.routePrefix }} + {{- end }} + access: proxy + jsonData: + handleGrafanaManagedAlerts: {{ .Values.grafana.sidecar.datasources.alertmanager.handleGrafanaManagedAlerts }} + implementation: {{ .Values.grafana.sidecar.datasources.alertmanager.implementation }} +{{- end }} +{{- end }} +{{- if .Values.grafana.additionalDataSources }} +{{ tpl (toYaml .Values.grafana.additionalDataSources | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml new file mode 100644 index 0000000000..dfc26d7ecd --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml @@ -0,0 +1,616 @@ +{{- /* +Generated from 'alertmanager-overview' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "alertmanager-overview" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + alertmanager-overview.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "current set of alerts stored in the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(alertmanager_alerts{namespace=~\"$namespace\",service=~\"$service\"}) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Alerts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "rate of successful and invalid alerts received by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(alertmanager_alerts_received_total{namespace=~\"$namespace\",service=~\"$service\"}[$__rate_interval])) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Received", + "refId": "A" + }, + { + "expr": "sum(rate(alertmanager_alerts_invalid_total{namespace=~\"$namespace\",service=~\"$service\"}[$__rate_interval])) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Invalid", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Alerts receive rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Alerts", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "rate of successful and invalid notifications sent by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "integration", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(alertmanager_notifications_total{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (integration,namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Total", + "refId": "A" + }, + { + "expr": "sum(rate(alertmanager_notifications_failed_total{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (integration,namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Failed", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "$integration: Notifications Send Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "latency of notifications sent by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "integration", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99,\n sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (le,namespace,service,instance)\n) \n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} 99th Percentile", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.50,\n sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (le,namespace,service,instance)\n) \n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Median", + "refId": "B" + }, + { + "expr": "sum(rate(alertmanager_notification_latency_seconds_sum{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (namespace,service,instance)\n/\nsum(rate(alertmanager_notification_latency_seconds_count{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (namespace,service,instance)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Average", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "$integration: Notification Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Notifications", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "alertmanager-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(alertmanager_alerts, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "service", + "multi": false, + "name": "service", + "options": [ + + ], + "query": "label_values(alertmanager_alerts, service)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "all", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": true, + "label": null, + "multi": false, + "name": "integration", + "options": [ + + ], + "query": "label_values(alertmanager_notifications_total{integration=~\".*\"}, integration)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Alertmanager / Overview", + "uid": "alertmanager-overview", + "version": 0 + } +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/apiserver.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/apiserver.yaml new file mode 100644 index 0000000000..bd1048b567 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/apiserver.yaml @@ -0,0 +1,1772 @@ +{{- /* +Generated from 'apiserver' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.kubeApiServer.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "apiserver" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + apiserver.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "content": "The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.", + "datasource": null, + "description": "The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.", + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "mode": "markdown", + "span": 12, + "title": "Notice", + "type": "text" + } + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of requests (both read and write) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"all\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Availability (30d) > 99.000%", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 3, + "description": "How much error budget is left looking at our 0.990% availability guarantees?", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100 * (apiserver_request:availability30d{verb=\"all\", cluster=\"$cluster\"} - 0.990000)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "errorbudget", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ErrorBudget (30d) > 99.000%", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "decimals": 3, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 3, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of read requests (LIST,GET) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"read\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Read Availability (30d)", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many read requests (LIST,GET) per second do the apiservers get by code?", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/2../i", + "color": "#56A64B" + }, + { + "alias": "/3../i", + "color": "#F2CC0C" + }, + { + "alias": "/4../i", + "color": "#3274D9" + }, + { + "alias": "/5../i", + "color": "#E02F44" + } + ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"read\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} code {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Requests", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many percent of read requests (LIST,GET) per second are returned with errors (5xx)?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"read\",code=~\"5..\", cluster=\"$cluster\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"read\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many seconds is the 99th percentile for reading (LIST|GET) a given resource?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster_quantile:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds:histogram_quantile{verb=\"read\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of write requests (POST|PUT|PATCH|DELETE) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"write\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Write Availability (30d)", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many write requests (POST|PUT|PATCH|DELETE) per second do the apiservers get by code?", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/2../i", + "color": "#56A64B" + }, + { + "alias": "/3../i", + "color": "#F2CC0C" + }, + { + "alias": "/4../i", + "color": "#3274D9" + }, + { + "alias": "/5../i", + "color": "#E02F44" + } + ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"write\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} code {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Requests", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many percent of write requests (POST|PUT|PATCH|DELETE) per second are returned with errors (5xx)?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"write\",code=~\"5..\", cluster=\"$cluster\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"write\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many seconds is the 99th percentile for writing (POST|PUT|PATCH|DELETE) a given resource?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster_quantile:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds:histogram_quantile{verb=\"write\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_adds_total{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Add Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_depth{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Depth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Latency", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 17, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"apiserver\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"apiserver\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / API server", + "uid": "09ec8aa1e996d6ffcd6817bbaff4db1b", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/cluster-total.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/cluster-total.yaml new file mode 100644 index 0000000000..f4be0bbd45 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/cluster-total.yaml @@ -0,0 +1,1882 @@ +{{- /* +Generated from 'cluster-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "cluster-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + cluster-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "Value #G", + "value": "Value #G" + }, + { + "text": "Value #H", + "value": "Value #H" + }, + { + "text": "namespace", + "value": "namespace" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "90%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Current Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/8b7a8b326d7a6f1f04244066368c67af/kubernetes-networking-namespace-pods?orgId=1&refresh=30s&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 6, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 9, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth History", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 15, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "What is TCP Retransmit?", + "url": "https://accedian.com/enterprises/blog/network-packet-loss-retransmissions-and-duplicate-acknowledgements/" + } + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(rate(node_netstat_Tcp_RetransSegs{cluster=\"$cluster\"}[$interval:$resolution]) / rate(node_netstat_Tcp_OutSegs{cluster=\"$cluster\"}[$interval:$resolution])) by (instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of TCP Retransmits out of all sent segments", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "Why monitor SYN retransmits?", + "url": "https://github.com/prometheus/node_exporter/issues/1023#issuecomment-408128365" + } + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(rate(node_netstat_TcpExt_TCPSynRetrans{cluster=\"$cluster\"}[$interval:$resolution]) / rate(node_netstat_Tcp_RetransSegs{cluster=\"$cluster\"}[$interval:$resolution])) by (instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of TCP SYN Retransmits out of all retransmits", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Cluster", + "uid": "ff635a025bcfea7bc3dd4f508990a3e9", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/controller-manager.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/controller-manager.yaml new file mode 100644 index 0000000000..8d420d7a4f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/controller-manager.yaml @@ -0,0 +1,1196 @@ +{{- /* +Generated from 'controller-manager' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeControllerManager.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "controller-manager" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + controller-manager.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + {{- if .Values.k3sServer.enabled }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", metrics_path=\"/metrics\"})", + {{- else }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"})", + {{- end }} + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_adds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Add Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_depth{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Depth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Latency", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Controller Manager", + "uid": "72e0e05bef5099e5f049b05fdc429ed4", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/etcd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/etcd.yaml new file mode 100644 index 0000000000..0eeedc6299 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/etcd.yaml @@ -0,0 +1,1229 @@ +{{- /* +Generated from 'etcd' from https://github.com/etcd-io/etcd.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeEtcd.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "etcd" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + etcd.json: |- + { + "annotations": { + "list": [] + }, + "description": "etcd sample Grafana dashboard with Prometheus", + "editable": true, + "gnetId": null, + "hideControls": false, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 28, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum(etcd_server_has_leader{job=\"$cluster\"})", + "intervalFactor": 2, + "legendFormat": "", + "metric": "etcd_server_has_leader", + "refId": "A", + "step": 20 + } + ], + "thresholds": "", + "title": "Up", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 23, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{job=\"$cluster\",grpc_type=\"unary\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Rate", + "metric": "grpc_server_started_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(rate(grpc_server_handled_total{job=\"$cluster\",grpc_type=\"unary\",grpc_code=~\"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Failed Rate", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 41, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", + "intervalFactor": 2, + "legendFormat": "Watch Streams", + "metric": "grpc_server_handled_total", + "refId": "A", + "step": 4 + }, + { + "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", + "intervalFactor": 2, + "legendFormat": "Lease Streams", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "showTitle": false, + "title": "Row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "etcd_mvcc_db_total_size_in_bytes{job=\"$cluster\"}", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} DB Size", + "metric": "", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "DB Size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=\"$cluster\"}[$__rate_interval])) by (instance, le))", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} WAL fsync", + "metric": "etcd_disk_wal_fsync_duration_seconds_bucket", + "refId": "A", + "step": 4 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=\"$cluster\"}[$__rate_interval])) by (instance, le))", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} DB fsync", + "metric": "etcd_disk_backend_commit_duration_seconds_bucket", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 29, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"$cluster\"}", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Resident Memory", + "metric": "process_resident_memory_bytes", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 5, + "id": 22, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(etcd_network_client_grpc_received_bytes_total{job=\"$cluster\"}[$__rate_interval])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Client Traffic In", + "metric": "etcd_network_client_grpc_received_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic In", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 5, + "id": 21, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(etcd_network_client_grpc_sent_bytes_total{job=\"$cluster\"}[$__rate_interval])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Client Traffic Out", + "metric": "etcd_network_client_grpc_sent_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic Out", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 20, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_peer_received_bytes_total{job=\"$cluster\"}[$__rate_interval])) by (instance)", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer Traffic In", + "metric": "etcd_network_peer_received_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Peer Traffic In", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_peer_sent_bytes_total{job=\"$cluster\"}[$__rate_interval])) by (instance)", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer Traffic Out", + "metric": "etcd_network_peer_sent_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Peer Traffic Out", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 40, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_failed_total{job=\"$cluster\"}[$__rate_interval]))", + "intervalFactor": 2, + "legendFormat": "Proposal Failure Rate", + "metric": "etcd_server_proposals_failed_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(etcd_server_proposals_pending{job=\"$cluster\"})", + "intervalFactor": 2, + "legendFormat": "Proposal Pending Total", + "metric": "etcd_server_proposals_pending", + "refId": "B", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_committed_total{job=\"$cluster\"}[$__rate_interval]))", + "intervalFactor": 2, + "legendFormat": "Proposal Commit Rate", + "metric": "etcd_server_proposals_committed_total", + "refId": "C", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total{job=\"$cluster\"}[$__rate_interval]))", + "intervalFactor": 2, + "legendFormat": "Proposal Apply Rate", + "refId": "D", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": 0, + "editable": true, + "error": false, + "fill": 0, + "id": 19, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "changes(etcd_server_leader_changes_seen_total{job=\"$cluster\"}[1d])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Total Leader Elections Per Day", + "metric": "etcd_server_leader_changes_seen_total", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total Leader Elections Per Day", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 0, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 42, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (instance, le) (rate(etcd_network_peer_round_trip_time_seconds_bucket{job=\"$cluster\"}[$__rate_interval])))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer round trip time", + "metric": "etcd_network_peer_round_trip_time_seconds_bucket", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Peer round trip time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:925", + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:926", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "New row" + } + ], + "schemaVersion": 13, + "sharedCrosshair": false, + "style": "dark", + "tags": [ + "etcd-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": {{ if (or .Values.grafana.sidecar.dashboards.multicluster.global.enabled .Values.grafana.sidecar.dashboards.multicluster.etcd.enabled) }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(etcd_server_has_leader, job)", + "refresh": 2, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "etcd", + "uid": "c2f4e12cdf69feb95caa41a5a1b423d9", + "version": 215 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/grafana-overview.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/grafana-overview.yaml new file mode 100644 index 0000000000..d2609140cf --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/grafana-overview.yaml @@ -0,0 +1,635 @@ +{{- /* +Generated from 'grafana-overview' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "grafana-overview" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + grafana-overview.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [ + + ], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 3085, + "iteration": 1631554945276, + "links": [ + + ], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "mappings": [ + + ], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": { + + }, + "textMode": "auto" + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "grafana_alerting_result_total{job=~\"$job\", instance=~\"$instance\", state=\"alerting\"}", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Firing Alerts", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": { + + }, + "textMode": "auto" + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "sum(grafana_stat_totals_dashboard{job=~\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Dashboards", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "displayMode": "auto" + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 10, + "options": { + "showHeader": true + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "grafana_build_info{job=~\"$job\", instance=~\"$instance\"}", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Build Info", + "transformations": [ + { + "id": "labelsToFields", + "options": { + + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "branch": true, + "container": true, + "goversion": true, + "namespace": true, + "pod": true, + "revision": true + }, + "indexByName": { + "Time": 7, + "Value": 11, + "branch": 4, + "container": 8, + "edition": 2, + "goversion": 6, + "instance": 1, + "job": 0, + "namespace": 9, + "pod": 10, + "revision": 5, + "version": 3 + }, + "renameByName": { + + } + } + } + ], + "type": "table" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ] + }, + "overrides": [ + + ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status_code) (irate(grafana_http_request_duration_seconds_count{job=~\"$job\", instance=~\"$instance\"}[1m])) ", + "interval": "", + "legendFormat": "{{`{{`}}status_code{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "RPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "$$hashKey": "object:157", + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:158", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ] + }, + "overrides": [ + + ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) by (le)) * 1", + "interval": "", + "legendFormat": "99th Percentile", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.50, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) by (le)) * 1", + "interval": "", + "legendFormat": "50th Percentile", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(irate(grafana_http_request_duration_seconds_sum{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) * 1 / sum(irate(grafana_http_request_duration_seconds_count{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Average", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "Request Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "$$hashKey": "object:210", + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:211", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 30, + "style": "dark", + "tags": [ + + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": [ + "default/grafana" + ], + "value": [ + "default/grafana" + ] + }, + "datasource": "$datasource", + "definition": "label_values(grafana_build_info, job)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "job", + "options": [ + + ], + "query": { + "query": "label_values(grafana_build_info, job)", + "refId": "Billing Admin-job-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "definition": "label_values(grafana_build_info, instance)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "instance", + "options": [ + + ], + "query": { + "query": "label_values(grafana_build_info, instance)", + "refId": "Billing Admin-instance-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Grafana Overview", + "uid": "6be0s85Mk", + "version": 2 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-coredns.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-coredns.yaml new file mode 100644 index 0000000000..7ecca76f23 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-coredns.yaml @@ -0,0 +1,1534 @@ +{{- /* +Generated from 'k8s-coredns' from ../files/dashboards/k8s-coredns.json +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.coreDns.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-coredns" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-coredns.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A dashboard for the CoreDNS DNS server with updated metrics for version 1.7.0+. Based on the CoreDNS dashboard by buhay.", + "editable": true, + "gnetId": 12539, + "graphTooltip": 0, + "iteration": 1603798405693, + "links": [ + { + "icon": "external link", + "tags": [], + "targetBlank": true, + "title": "CoreDNS.io", + "type": "link", + "url": "https://coredns.io" + } + ], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (proto) or\nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (proto)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (total)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + }, + { + "alias": "other", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_type_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type) or \nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{type}}"}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (by qtype)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (zone) or\nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (zone)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{zone}}"}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (by zone)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_do_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) or\nsum(rate(coredns_dns_do_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "DO", + "refId": "A", + "step": 40 + }, + { + "expr": "sum(rate(coredns_dns_request_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) or\nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "total", + "refId": "B", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (DO bit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "tcp:90", + "yaxis": 2 + }, + { + "alias": "tcp:99 ", + "yaxis": 2 + }, + { + "alias": "tcp:50", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99 ", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90", + "refId": "B", + "step": 60 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50", + "refId": "C", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (size, udp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 7 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "tcp:90", + "yaxis": 1 + }, + { + "alias": "tcp:99 ", + "yaxis": 1 + }, + { + "alias": "tcp:50", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99 ", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90", + "refId": "B", + "step": 60 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50", + "refId": "C", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (size,tcp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_response_rcode_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (rcode) or\nsum(rate(coredns_dns_responses_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (rcode)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{rcode}}"}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (by rcode)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 14 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_duration_seconds_bucket{job=\"coredns\",instance=~\"$instance\"}[5m])) by (le, job))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_duration_seconds_bucket{job=\"coredns\",instance=~\"$instance\"}[5m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_duration_seconds_bucket{job=\"coredns\",instance=~\"$instance\"}[5m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "50%", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (duration)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "udp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:50%", + "yaxis": 2 + }, + { + "alias": "tcp:90%", + "yaxis": 2 + }, + { + "alias": "tcp:99%", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50%", + "metric": "", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (size, udp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "udp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:90%", + "yaxis": 1 + }, + { + "alias": "tcp:99%", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le, proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50%", + "metric": "", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (size, tcp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(coredns_cache_size{job=\"coredns\",instance=~\"$instance\"}) by (type) or\nsum(coredns_cache_entries{job=\"coredns\",instance=~\"$instance\"}) by (type)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{type}}"}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cache (size)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 28 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "misses", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_cache_hits_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type)", + "hide": false, + "intervalFactor": 2, + "legendFormat": "hits:{{"{{type}}"}}", + "refId": "A", + "step": 40 + }, + { + "expr": "sum(rate(coredns_cache_misses_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type)", + "hide": false, + "intervalFactor": 2, + "legendFormat": "misses", + "refId": "B", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cache (hitrate)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "dns", + "coredns" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "definition": "label_values(up{job=\"coredns\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(up{job=\"coredns\"}, instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "CoreDNS", + "uid": "vkQ0UHxik", + "version": 2 + } +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml new file mode 100644 index 0000000000..93ee57db93 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml @@ -0,0 +1,3088 @@ +{{- /* +Generated from 'k8s-resources-cluster' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-cluster" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-cluster.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster:node_cpu:ratio_rate5m{cluster=\"$cluster\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Requests Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Limits Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(:node_memory_MemAvailable_bytes:sum{cluster=\"$cluster\"}) / sum(node_memory_MemTotal_bytes{job=\"node-exporter\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Requests Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Limits Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to workloads", + "linkUrl": "d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kube_pod_owner{job=\"kube-state-metrics\", cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to workloads", + "linkUrl": "d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kube_pod_owner{job=\"kube-state-metrics\", cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Requests by Namespace", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Requests", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Namespace: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Namespace: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Namespace", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 17, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 19, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 20, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 21, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 22, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Cluster", + "uid": "efa86fd1d0c121a26444b636a3f509a8", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml new file mode 100644 index 0000000000..9c295831a5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-multicluster' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-multicluster" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-multicluster.json: |- + {{`{"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"links":[],"refresh":"10s","rows":[{"collapse":false,"height":"100px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":1,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"cluster:node_cpu:ratio_rate5m","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":2,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"cpu\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Requests Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":3,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"cpu\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Limits Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(node_memory_MemTotal_bytes{job=\"node-exporter\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":5,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"memory\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Requests Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":6,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"memory\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Limits Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Headlines","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"id":7,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster)","format":"time_series","legendFormat":"{{cluster}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":8,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Cluster","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/efa86fd1d0c121a26444b636a3f509a8/k8s-resources-cluster?var-datasource=$datasource&var-cluster=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"cluster","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"id":9,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster)","format":"time_series","legendFormat":"{{cluster}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage (w/o cache)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":10,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Cluster","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/efa86fd1d0c121a26444b636a3f509a8/k8s-resources-cluster?var-datasource=$datasource&var-cluster=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"cluster","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Requests by Cluster","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Requests","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Multi-Cluster","uid":"b59e6c9f2fcbe2e16d77fc492374cc4f","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml new file mode 100644 index 0000000000..1c32c9c02e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml @@ -0,0 +1,2797 @@ +{{- /* +Generated from 'k8s-resources-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation (from requests)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation (from limits)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation (from requests)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation (from limits)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_cache{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(container_memory_swap{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 17, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Namespace (Pods)", + "uid": "85a562078cdf77779eaa1add43ccec1e", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml new file mode 100644 index 0000000000..e60a42d747 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml @@ -0,0 +1,1026 @@ +{{- /* +Generated from 'k8s-resources-node' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-node" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-node.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max capacity", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max capacity", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max capacity", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max capacity", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\", container!=\"\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_rss{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_cache{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_swap{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": true, + "name": "node", + "options": [ + + ], + "query": "label_values(kube_node_info{cluster=\"$cluster\"}, node)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Node (Pods)", + "uid": "200ac8fdbfbb74b39aff88118e4d1c2c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml new file mode 100644 index 0000000000..80fab51c00 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml @@ -0,0 +1,2469 @@ +{{- /* +Generated from 'k8s-resources-pod' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-pod.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "requests", + "color": "#F2495C", + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "limits", + "color": "#FF9830", + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(container_cpu_cfs_throttled_periods_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container) /sum(increase(container_cpu_cfs_periods_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.25, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Throttling", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Throttling", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (WSS)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage (WSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_cache{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(container_memory_swap{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Reads", + "legendLink": null, + "step": 10 + }, + { + "expr": "ceil(sum by(pod) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Writes", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Reads", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Writes", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution(Pod - Read & Writes)", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(container) (rate(container_fs_reads_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution(Containers)", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(container) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\",device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "options": [ + + ], + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Pod", + "uid": "6581e46e4e5c7ba40a07646395ef7b23", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml new file mode 100644 index 0000000000..d77170afd8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-windows-cluster' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-windows-cluster" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-windows-cluster.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"100px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"1 - avg(rate(windows_cpu_time_total{cluster=\"$cluster\", job=\"windows-exporter\", mode=\"idle\"}[1m]))","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\"}) / sum(node:windows_node_num_cpu:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Requests Commitment","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\"}) / sum(node:windows_node_num_cpu:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Limits Commitment","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"1 - sum(:windows_node_memory_MemFreeCached_bytes:sum{cluster=\"$cluster\"}) / sum(:windows_node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\"}) / sum(:windows_node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Requests Commitment","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":7,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\"}) / sum(:windows_node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Limits Commitment","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Headlines","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":8,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":9,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Namespace","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/490b402361724ab1d4c45666c1fa9b6f/k8s-resources-windows-namespace?var-datasource=$datasource&var-namespace=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"namespace","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":10,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Private Working Set)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"decbytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":11,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Namespace","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/490b402361724ab1d4c45666c1fa9b6f/k8s-resources-windows-namespace?var-datasource=$datasource&var-namespace=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"namespace","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Requests by Namespace","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Requests","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Cluster(Windows)","uid":"4d08557fd9391b100730f2494bccac68","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml new file mode 100644 index 0000000000..13a1fc3abd --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-windows-namespace' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-windows-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-windows-namespace.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/40597a704a610e936dc6ed374a7ce023/k8s-resources-windows-pod?var-datasource=$datasource&var-namespace=$namespace&var-pod=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"decbytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/40597a704a610e936dc6ed374a7ce023/k8s-resources-windows-pod?var-datasource=$datasource&var-namespace=$namespace&var-pod=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Quota","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Namespace","multi":false,"name":"namespace","options":[],"query":"label_values(windows_pod_container_available{cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Namespace(Windows)","uid":"490b402361724ab1d4c45666c1fa9b6f","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml new file mode 100644 index 0000000000..6686e54053 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-windows-pod' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-windows-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-windows-pod.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"time_series","legendFormat":"{{container}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Container","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"container","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"time_series","legendFormat":"{{container}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Container","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"container","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"id":6,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(sum by (container) (rate(windows_container_network_received_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[1m])))","format":"time_series","intervalFactor":2,"legendFormat":"Received : {{ container }}","refId":"A"},{"expr":"sort_desc(sum by (container) (rate(windows_container_network_transmitted_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[1m])))","format":"time_series","intervalFactor":2,"legendFormat":"Transmitted : {{ container }}","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network I/O","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Network I/O","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Namespace","multi":false,"name":"namespace","options":[],"query":"label_values(windows_pod_container_available{cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Pod","multi":false,"name":"pod","options":[],"query":"label_values(windows_pod_container_available{cluster=\"$cluster\",namespace=\"$namespace\"}, pod)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Pod(Windows)","uid":"40597a704a610e936dc6ed374a7ce023","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml new file mode 100644 index 0000000000..e2a63ae208 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml @@ -0,0 +1,2024 @@ +{{- /* +Generated from 'k8s-resources-workload' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-workload" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-workload.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Pod: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Pod: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Pod", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\"}, workload_type)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}, workload)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Workload", + "uid": "a164a7f0339f99e89cea5cb47e9be617", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml new file mode 100644 index 0000000000..95d758ea2d --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml @@ -0,0 +1,2189 @@ +{{- /* +Generated from 'k8s-resources-workloads-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-workloads-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-workloads-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}} - {{`{{`}}workload_type{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}} - {{`{{`}}workload_type{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$type", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Workload: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Workload: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Workload", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Namespace (Workloads)", + "uid": "a87fb0d919ec0ea5f6543124e16c42a5", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml new file mode 100644 index 0000000000..d9ce9d738c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-windows-cluster-rsrc-use' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-windows-cluster-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-windows-cluster-rsrc-use.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_cpu_utilisation:avg1m{cluster=\"$cluster\"} * node:windows_node_num_cpu:sum{cluster=\"$cluster\"} / scalar(sum(node:windows_node_num_cpu:sum{cluster=\"$cluster\"}))","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_memory_utilisation:ratio{cluster=\"$cluster\"}","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_memory_swap_io_pages:irate{cluster=\"$cluster\"}","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Saturation (Swap I/O Pages)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_disk_utilisation:avg_irate{cluster=\"$cluster\"} / scalar(node:windows_node:sum{cluster=\"$cluster\"})","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk IO Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_net_utilisation:sum_irate{cluster=\"$cluster\"}","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Net Utilisation (Transmitted)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":7,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_net_saturation:sum_irate{cluster=\"$cluster\"}","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Net Saturation (Dropped)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Network","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":8,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum by (instance)(node:windows_node_filesystem_usage:{cluster=\"$cluster\"})\n","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk Capacity","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Storage","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / USE Method / Cluster(Windows)","uid":"53a43377ec9aaf2ff64dfc7a1f539334","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml new file mode 100644 index 0000000000..a7608496a3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-windows-node-rsrc-use' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-windows-node-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-windows-node-rsrc-use.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_cpu_utilisation:avg1m{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Utilisation","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"sum by (core) (irate(windows_cpu_time_total{cluster=\"$cluster\", job=\"windows-exporter\", mode!=\"idle\", instance=\"$instance\"}[$__rate_interval]))","format":"time_series","legendFormat":"{{core}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage Per Core","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_memory_utilisation:{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Memory","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Utilisation %","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"id":5,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"max(\n windows_os_visible_memory_bytes{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}\n - windows_memory_available_bytes{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}\n)\n","format":"time_series","intervalFactor":2,"legendFormat":"memory used","refId":"A"},{"expr":"max(node:windows_node_memory_totalCached_bytes:sum{cluster=\"$cluster\", instance=\"$instance\"})","format":"time_series","intervalFactor":2,"legendFormat":"memory cached","refId":"B"},{"expr":"max(windows_memory_available_bytes{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"})","format":"time_series","intervalFactor":2,"legendFormat":"memory free","refId":"C"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_memory_swap_io_pages:irate{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Swap IO","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Saturation (Swap I/O) Pages","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":7,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_disk_utilisation:avg_irate{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Utilisation","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk IO Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"id":8,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"read","yaxis":1},{"alias":"io time","yaxis":2}],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"max(rate(windows_logical_disk_read_bytes_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]))","format":"time_series","intervalFactor":2,"legendFormat":"read","refId":"A"},{"expr":"max(rate(windows_logical_disk_write_bytes_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]))","format":"time_series","intervalFactor":2,"legendFormat":"written","refId":"B"},{"expr":"max(rate(windows_logical_disk_read_seconds_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]) + rate(windows_logical_disk_write_seconds_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]))","format":"time_series","intervalFactor":2,"legendFormat":"io time","refId":"C"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk I/O","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ms","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":9,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_net_utilisation:sum_irate{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Utilisation","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Net Utilisation (Transmitted)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":10,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_net_saturation:sum_irate{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Saturation","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Net Saturation (Dropped)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Net","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":11,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_filesystem_usage:{cluster=\"$cluster\", instance=\"$instance\"}\n","format":"time_series","legendFormat":"{{volume}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Instance","multi":false,"name":"instance","options":[],"query":"label_values(windows_system_system_up_time{cluster=\"$cluster\"}, instance)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / USE Method / Node(Windows)","uid":"96e7484b0bb53b74fbc2bcb7723cd40b","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/kubelet.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/kubelet.yaml new file mode 100644 index 0000000000..74a5303f8f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/kubelet.yaml @@ -0,0 +1,2256 @@ +{{- /* +Generated from 'kubelet' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubelet.enabled" .) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "kubelet" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + kubelet.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 2, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_node_name{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Running Kubelets", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 3, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_running_pods{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}) OR sum(kubelet_running_pod_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Running Pods", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 4, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_running_containers{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}) OR sum(kubelet_running_container_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Running Containers", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 5, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\", state=\"actual_state_of_world\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Actual Volume Count", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 6, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",state=\"desired_state_of_world\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Desired Volume Count", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 7, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(rate(kubelet_node_config_error{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Config Error Count", + "transparent": false, + "type": "stat" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_runtime_operations_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (operation_type, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_runtime_operations_errors_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_runtime_operations_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} pod", + "refId": "A" + }, + { + "expr": "sum(rate(kubelet_pod_worker_duration_seconds_count{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} worker", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pod Start Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} pod", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} worker", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pod Start Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(storage_operation_duration_seconds_count{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(storage_operation_errors_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_cgroup_manager_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Cgroup manager operation rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_cgroup_manager_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Cgroup manager 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Pod lifecycle event generator", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_pleg_relist_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_interval_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist interval", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 21, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 70 + }, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Request duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 77 + }, + "id": 23, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 77 + }, + "id": 24, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 77 + }, + "id": 25, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Kubelet", + "uid": "3138fa155d5915769fbded898ac09fd9", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml new file mode 100644 index 0000000000..f5c72844fb --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml @@ -0,0 +1,1464 @@ +{{- /* +Generated from 'namespace-by-pod' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "namespace-by-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + namespace-by-pod.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "height": 9, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "height": 9, + "id": 4, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "pod", + "value": "pod" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "100%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/7a18067ce943a40ae25454675c19ff5c/kubernetes-networking-pod?orgId=1&refresh=30s&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 6, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 9, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 30 + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Namespace (Pods)", + "uid": "8b7a8b326d7a6f1f04244066368c67af", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml new file mode 100644 index 0000000000..801b09c265 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml @@ -0,0 +1,1736 @@ +{{- /* +Generated from 'namespace-by-workload' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "namespace-by-workload" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + namespace-by-workload.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "Value #G", + "value": "Value #G" + }, + { + "text": "Value #H", + "value": "Value #H" + }, + { + "text": "workload", + "value": "workload" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "90%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Current Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/728bf77cc1166d2f3133bf25846876cc/kubernetes-networking-workload?orgId=1&refresh=30s&var-namespace=$namespace&var-type=$type&var-workload=$__cell", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 6, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 9, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth HIstory", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 15, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 17, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Namespace (Workload)", + "uid": "bbb2a765a623ae38130206c7d94a160f", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml new file mode 100644 index 0000000000..9869a3d3e0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml @@ -0,0 +1,1063 @@ +{{- /* +Generated from 'node-cluster-rsrc-use' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "node-cluster-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + node-cluster-rsrc-use.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "((\n instance:node_cpu_utilisation:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n *\n instance:node_num_cpu:sum{job=\"node-exporter\", cluster=\"$cluster\"}\n) != 0 )\n/ scalar(sum(instance:node_num_cpu:sum{job=\"node-exporter\", cluster=\"$cluster\"}))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} instance {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance:node_load1_per_cpu:ratio{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance:node_load1_per_cpu:ratio{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Saturation (Load1 per CPU)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance:node_memory_utilisation:ratio{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance:node_memory_utilisation:ratio{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_vmstat_pgmajfault:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Saturation (Major Page Faults)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/Receive/", + "stack": "A" + }, + { + "alias": "/Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Utilisation (Bytes Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ Receive/", + "stack": "A" + }, + { + "alias": "/ Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Saturation (Drops Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Saturation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk IO", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum without (device) (\n max without (fstype, mountpoint) ((\n node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"}\n -\n node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"}\n ) != 0)\n)\n/ scalar(sum(max without (fstype, mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"})))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk Space", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(node_time_seconds, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / USE Method / Cluster", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/node-rsrc-use.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/node-rsrc-use.yaml new file mode 100644 index 0000000000..75e69afa22 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/node-rsrc-use.yaml @@ -0,0 +1,1089 @@ +{{- /* +Generated from 'node-rsrc-use' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "node-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + node-rsrc-use.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_cpu_utilisation:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Utilisation", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_load1_per_cpu:ratio{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Saturation", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Saturation (Load1 per CPU)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_memory_utilisation:ratio{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Utilisation", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_vmstat_pgmajfault:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Major page Faults", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Saturation (Major Page Faults)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/Receive/", + "stack": "A" + }, + { + "alias": "/Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Utilisation (Bytes Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ Receive/", + "stack": "A" + }, + { + "alias": "/ Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Saturation (Drops Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Saturation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk IO", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(1 -\n (\n max without (mountpoint, fstype) (node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\", cluster=\"$cluster\"})\n /\n max without (mountpoint, fstype) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\", cluster=\"$cluster\"})\n ) != 0\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk Space", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(node_time_seconds, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_exporter_build_info{job=\"node-exporter\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / USE Method / Node", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/nodes-darwin.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/nodes-darwin.yaml new file mode 100644 index 0000000000..fe11875324 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/nodes-darwin.yaml @@ -0,0 +1,1073 @@ +{{- /* +Generated from 'nodes-darwin' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (and (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) .Values.nodeExporter.operatingSystems.darwin.enabled) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "nodes-darwin" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + nodes-darwin.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\", instance=\"$instance\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n", + "format": "time_series", + "intervalFactor": 5, + "legendFormat": "{{`{{`}}cpu{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "1m load average", + "refId": "A" + }, + { + "expr": "node_load5{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5m load average", + "refId": "B" + }, + { + "expr": "node_load15{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "15m load average", + "refId": "C" + }, + { + "expr": "count(node_cpu_seconds_total{job=\"node-exporter\", instance=\"$instance\", mode=\"idle\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "logical cores", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_total_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Physical Memory", + "refId": "A" + }, + { + "expr": "(\n node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"} -\n node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"} +\n node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"} +\n node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Memory Used", + "refId": "B" + }, + { + "expr": "(\n node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"} -\n node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "App Memory", + "refId": "C" + }, + { + "expr": "node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Wired Memory", + "refId": "D" + }, + { + "expr": "node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Compressed", + "refId": "E" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + + }, + "id": 5, + "span": 3, + "targets": [ + { + "expr": "(\n (\n avg(node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"}) -\n avg(node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"}) +\n avg(node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"}) +\n avg(node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"})\n ) /\n avg(node_memory_total_bytes{job=\"node-exporter\", instance=\"$instance\"})\n)\n*\n100\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Memory Usage", + "transparent": false, + "type": "gauge" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ read| written/", + "yaxis": 1 + }, + { + "alias": "/ io time/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} read", + "refId": "A" + }, + { + "expr": "rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} written", + "refId": "B" + }, + { + "expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} io time", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.8 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Mounted on" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Size" + }, + "properties": [ + { + "id": "custom.width", + "value": 93 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "custom.width", + "value": 72 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Available" + }, + "properties": [ + { + "id": "custom.width", + "value": 88 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used, %" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "max", + "value": 1 + }, + { + "id": "min", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + + }, + "id": 7, + "span": 6, + "targets": [ + { + "expr": "max by (mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + }, + { + "expr": "max by (mountpoint) (node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Disk Space Usage", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value #A": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #B": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "mountpoint": { + "aggregations": [ + + ], + "operation": "groupby" + } + } + } + }, + { + "id": "merge", + "options": { + + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used", + "binary": { + "left": "Value #A (lastNotNull)", + "operator": "-", + "reducer": "sum", + "right": "Value #B (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used, %", + "binary": { + "left": "Used", + "operator": "/", + "reducer": "sum", + "right": "Value #A (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + + }, + "indexByName": { + + }, + "renameByName": { + "Value #A (lastNotNull)": "Size", + "Value #B (lastNotNull)": "Available", + "mountpoint": "Mounted on" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": { + + }, + "sort": [ + { + "field": "Mounted on" + } + ] + } + } + ], + "transparent": false, + "type": "table" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network received (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network transmitted (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Transmitted", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_uname_info{job=\"node-exporter\", sysname=\"Darwin\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / MacOS", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/nodes.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/nodes.yaml new file mode 100644 index 0000000000..0da40a7b99 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/nodes.yaml @@ -0,0 +1,1066 @@ +{{- /* +Generated from 'nodes' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (and (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) .Values.nodeExporter.operatingSystems.linux.enabled) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "nodes" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + nodes.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\", instance=\"$instance\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n", + "format": "time_series", + "intervalFactor": 5, + "legendFormat": "{{`{{`}}cpu{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "1m load average", + "refId": "A" + }, + { + "expr": "node_load5{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5m load average", + "refId": "B" + }, + { + "expr": "node_load15{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "15m load average", + "refId": "C" + }, + { + "expr": "count(node_cpu_seconds_total{job=\"node-exporter\", instance=\"$instance\", mode=\"idle\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "logical cores", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory used", + "refId": "A" + }, + { + "expr": "node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory buffers", + "refId": "B" + }, + { + "expr": "node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory cached", + "refId": "C" + }, + { + "expr": "node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory free", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + + }, + "id": 5, + "span": 3, + "targets": [ + { + "expr": "100 -\n(\n avg(node_memory_MemAvailable_bytes{job=\"node-exporter\", instance=\"$instance\"}) /\n avg(node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"})\n* 100\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Memory Usage", + "transparent": false, + "type": "gauge" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ read| written/", + "yaxis": 1 + }, + { + "alias": "/ io time/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} read", + "refId": "A" + }, + { + "expr": "rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} written", + "refId": "B" + }, + { + "expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} io time", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.8 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Mounted on" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Size" + }, + "properties": [ + { + "id": "custom.width", + "value": 93 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "custom.width", + "value": 72 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Available" + }, + "properties": [ + { + "id": "custom.width", + "value": 88 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used, %" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "max", + "value": 1 + }, + { + "id": "min", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + + }, + "id": 7, + "span": 6, + "targets": [ + { + "expr": "max by (mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + }, + { + "expr": "max by (mountpoint) (node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Disk Space Usage", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value #A": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #B": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "mountpoint": { + "aggregations": [ + + ], + "operation": "groupby" + } + } + } + }, + { + "id": "merge", + "options": { + + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used", + "binary": { + "left": "Value #A (lastNotNull)", + "operator": "-", + "reducer": "sum", + "right": "Value #B (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used, %", + "binary": { + "left": "Used", + "operator": "/", + "reducer": "sum", + "right": "Value #A (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + + }, + "indexByName": { + + }, + "renameByName": { + "Value #A (lastNotNull)": "Size", + "Value #B (lastNotNull)": "Available", + "mountpoint": "Mounted on" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": { + + }, + "sort": [ + { + "field": "Mounted on" + } + ] + } + } + ], + "transparent": false, + "type": "table" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network received (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network transmitted (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Transmitted", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_uname_info{job=\"node-exporter\", sysname!=\"Darwin\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / Nodes", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml new file mode 100644 index 0000000000..4d1e33208b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml @@ -0,0 +1,587 @@ +{{- /* +Generated from 'persistentvolumesusage' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "persistentvolumesusage" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + persistentvolumesusage.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n sum without(instance, node) (topk(1, (kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n -\n sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used Space", + "refId": "A" + }, + { + "expr": "sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Free Space", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume Space Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max without(instance,node) (\n(\n topk(1, kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n topk(1, kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n/\ntopk(1, kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n* 100)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume Space Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used inodes", + "refId": "A" + }, + { + "expr": "(\n sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n -\n sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": " Free inodes", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume inodes Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max without(instance,node) (\ntopk(1, kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n/\ntopk(1, kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n* 100)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume inodes Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "PersistentVolumeClaim", + "multi": false, + "name": "volume", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\"}, persistentvolumeclaim)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Persistent Volumes", + "uid": "919b92a8e8041bd567af9edab12c840c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/pod-total.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/pod-total.yaml new file mode 100644 index 0000000000..9a7e7d0603 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/pod-total.yaml @@ -0,0 +1,1228 @@ +{{- /* +Generated from 'pod-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "pod-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + pod-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "height": 9, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace: $pod", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "height": 9, + "id": 4, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace: $pod", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 8, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}, pod)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Pod", + "uid": "7a18067ce943a40ae25454675c19ff5c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml new file mode 100644 index 0000000000..5c11900e69 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml @@ -0,0 +1,1674 @@ +{{- /* +Generated from 'prometheus-remote-write' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.prometheus.prometheusSpec.remoteWriteDashboards }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "prometheus-remote-write" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + prometheus-remote-write.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "60s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(\n prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} \n- \n ignoring(remote_name, url) group_right(instance) (prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} != 0)\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Highest Timestamp In vs. Highest Timestamp Sent", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "clamp_min(\n rate(prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) \n- \n ignoring (remote_name, url) group_right(instance) rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n, 0)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate[5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Timestamps", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(\n prometheus_remote_storage_samples_in_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n- \n ignoring(remote_name, url) group_right(instance) (rate(prometheus_remote_storage_succeeded_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]))\n- \n (rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate, in vs. succeeded or dropped [5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Samples", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_max{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Max Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_min{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Min Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_desired{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Desired Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Shards", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shard_capacity{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Shard Capacity", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_pending_samples{cluster=~\"$cluster\", instance=~\"$instance\"} or prometheus_remote_storage_samples_pending{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pending Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Shard Details", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_wal_segment_current{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "TSDB Current Segment", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_wal_watcher_current_segment{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}consumer{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Remote Write Current Segment", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Segments", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Dropped Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_failed_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Failed Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_retried_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_retried_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Retried Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_enqueue_retries_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Enqueue Retries", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Misc. Rates", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "prometheus-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "value": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": true, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_container_info{image=~\".*prometheus.*\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "value": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(prometheus_build_info{cluster=~\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "url", + "options": [ + + ], + "query": "label_values(prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}, url)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Prometheus / Remote Write", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/prometheus.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/prometheus.yaml new file mode 100644 index 0000000000..27f7c44e2c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/prometheus.yaml @@ -0,0 +1,1235 @@ +{{- /* +Generated from 'prometheus' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "prometheus" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + prometheus.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "60s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Count", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Uptime", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "s" + }, + { + "alias": "Instance", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "instance", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Job", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "job", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Version", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "version", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count by (job, instance, version) (prometheus_build_info{job=~\"$job\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "max by (job, instance) (time() - process_start_time_seconds{job=~\"$job\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Prometheus Stats", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Prometheus Stats", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(prometheus_target_sync_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[5m])) by (scrape_job) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}scrape_job{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Target Sync", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(prometheus_sd_discovered_targets{job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Targets", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Targets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Discovery", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_target_interval_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[5m]) / rate(prometheus_target_interval_length_seconds_count{job=~\"$job\",instance=~\"$instance\"}[5m]) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}interval{{`}}`}} configured", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Scrape Interval Duration", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_exceeded_body_size_limit_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "exceeded body size limit: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_exceeded_sample_limit_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "exceeded sample limit: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "duplicate timestamp: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_bounds_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "out of bounds: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_order_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "out of order: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scrape failures", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=~\"$job\",instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Appended Samples", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Retrieval", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_series{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}} head series", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Head Series", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_chunks{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}} head chunks", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Head Chunks", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_engine_query_duration_seconds_count{job=~\"$job\",instance=~\"$instance\",slice=\"inner_eval\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Query Rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "max by (slice) (prometheus_engine_query_duration_seconds{quantile=\"0.9\",job=~\"$job\",instance=~\"$instance\"}) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}slice{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Stage Duration", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Query", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "prometheus-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "job", + "multi": true, + "name": "job", + "options": [ + + ], + "query": "label_values(prometheus_build_info{job=\"prometheus-k8s\",namespace=\"monitoring\"}, job)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [ + + ], + "query": "label_values(prometheus_build_info{job=~\"$job\"}, instance)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Prometheus / Overview", + "uid": "", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/proxy.yaml new file mode 100644 index 0000000000..410812451e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/proxy.yaml @@ -0,0 +1,1276 @@ +{{- /* +Generated from 'proxy' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeProxy.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "proxy" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + proxy.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + {{- if .Values.k3sServer.enabled }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", metrics_path=\"/metrics\"})", + {{- else }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\"})", + {{- end }} + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubeproxy_sync_proxy_rules_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "rate", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rules Sync Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99,rate(kubeproxy_sync_proxy_rules_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rule Sync Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubeproxy_network_programming_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "rate", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Programming Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Programming Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\",verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeProxy.jobName" . }}\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeProxy.jobName" . }}\", cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Proxy", + "uid": "632e265de029684c40b21cb76bca4f94", + "version": 0 + } +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/scheduler.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/scheduler.yaml new file mode 100644 index 0000000000..ee0cf08b2f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/scheduler.yaml @@ -0,0 +1,1118 @@ +{{- /* +Generated from 'scheduler' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeScheduler.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "scheduler" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + scheduler.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + {{- if .Values.k3sServer.enabled }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", metrics_path=\"/metrics\"})", + {{- else }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\"})", + {{- end }} + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(scheduler_e2e_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} e2e", + "refId": "A" + }, + { + "expr": "sum(rate(scheduler_binding_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} binding", + "refId": "B" + }, + { + "expr": "sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} scheduling algorithm", + "refId": "C" + }, + { + "expr": "sum(rate(scheduler_volume_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} volume", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scheduling Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} e2e", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} binding", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} scheduling algorithm", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} volume", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scheduling latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Scheduler", + "uid": "2e6b6a3b4bddf1427b3a55aa1311c656", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/workload-total.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/workload-total.yaml new file mode 100644 index 0000000000..5aafccdebe --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/dashboards-1.14/workload-total.yaml @@ -0,0 +1,1438 @@ +{{- /* +Generated from 'workload-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "workload-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + workload-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 8, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth HIstory", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 14, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\"}, workload)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\"}, workload)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Workload", + "uid": "728bf77cc1166d2f3133bf25846876cc", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/namespaces.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/namespaces.yaml new file mode 100644 index 0000000000..39ed210ed4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/grafana/namespaces.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled (not .Values.grafana.defaultDashboards.useExistingNamespace) }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Values.grafana.defaultDashboards.namespace }} + labels: + name: {{ .Values.grafana.defaultDashboards.namespace }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + annotations: +{{- if not .Values.grafana.defaultDashboards.cleanupOnUninstall }} + helm.sh/resource-policy: "keep" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/_prometheus-operator.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/_prometheus-operator.tpl new file mode 100644 index 0000000000..6ae9dc72e6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/_prometheus-operator.tpl @@ -0,0 +1,7 @@ +{{/* Generate basic labels for prometheus-operator */}} +{{- define "kube-prometheus-stack.prometheus-operator.labels" }} +{{- include "kube-prometheus-stack.labels" . }} +app: {{ template "kube-prometheus-stack.name" . }}-operator +app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus-operator +app.kubernetes.io/component: prometheus-operator +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl new file mode 100644 index 0000000000..f419caf54b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl @@ -0,0 +1,6 @@ +{{/* Generate basic labels for prometheus-operator-webhook */}} +{{- define "kube-prometheus-stack.prometheus-operator-webhook.labels" }} +{{- include "kube-prometheus-stack.labels" . }} +app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus-operator +app.kubernetes.io/component: prometheus-operator-webhook +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml new file mode 100644 index 0000000000..054eac4a77 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml @@ -0,0 +1,143 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-webhook + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.labels }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.annotations | indent 4 }} +{{- end }} +spec: + replicas: {{ .Values.prometheusOperator.admissionWebhooks.deployment.replicas }} + revisionHistoryLimit: {{ .Values.prometheusOperator.admissionWebhooks.deployment.revisionHistoryLimit }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.strategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + release: {{ $.Release.Name | quote }} + template: + metadata: + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 8 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.podLabels }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.podLabels | indent 8 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.podAnnotations }} + annotations: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.podAnnotations | indent 8 }} +{{- end }} + spec: + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.admissionWebhooks.deployment.priorityClassName }} + {{- end }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "kube-prometheus-stack.imagePullSecrets" . | indent 8 }} + {{- end }} + containers: + - name: prometheus-operator-admission-webhook + {{- $operatorRegistry := .Values.global.imageRegistry | default .Values.prometheusOperator.admissionWebhooks.deployment.image.registry -}} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.image.sha }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.sha }}" + {{- else }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: "{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.pullPolicy }}" + args: + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.logFormat }} + - --log-format={{ .Values.prometheusOperator.admissionWebhooks.deployment.logFormat }} + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.logLevel }} + - --log-level={{ .Values.prometheusOperator.admissionWebhooks.deployment.logLevel }} + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + - "--web.enable-tls=true" + - "--web.cert-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.crt{{ else }}cert{{ end }}" + - "--web.key-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.key{{ else }}key{{ end }}" + - "--web.listen-address=:{{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.internalPort }}" + - "--web.tls-min-version={{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.tlsMinVersion }}" + ports: + - containerPort: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.internalPort }} + name: https + {{- else }} + ports: + - containerPort: 8080 + name: http + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /healthz + port: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "https" "http" }} + scheme: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "HTTPS" "HTTP" }} + initialDelaySeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.failureThreshold }} + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /healthz + port: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "https" "http" }} + scheme: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "HTTPS" "HTTP" }} + initialDelaySeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.failureThreshold }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.resources | indent 12 }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.containerSecurityContext | indent 12 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + volumeMounts: + - name: tls-secret + mountPath: /cert + readOnly: true + volumes: + - name: tls-secret + secret: + defaultMode: 420 + secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.dnsConfig }} + dnsConfig: +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.securityContext | indent 8 }} +{{- end }} + serviceAccountName: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }}-webhook + automountServiceAccountToken: {{ .Values.prometheusOperator.admissionWebhooks.deployment.automountServiceAccountToken }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.hostNetwork }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml new file mode 100644 index 0000000000..52dd78f624 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml @@ -0,0 +1,15 @@ +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.podDisruptionBudget -}} +apiVersion: policy/v1{{ ternary "" "beta1" ($.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget") }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-webhook + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + release: {{ $.Release.Name | quote }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.podDisruptionBudget | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/service.yaml new file mode 100644 index 0000000000..b06c129123 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/service.yaml @@ -0,0 +1,58 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-webhook + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.labels }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.clusterIP }} + clusterIP: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.clusterIP }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheusOperator.admissionWebhooks.deployment.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.externalTrafficPolicy }} +{{- end }} + ports: + {{- if not .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + - name: http + {{- if eq .Values.prometheusOperator.admissionWebhooks.deployment.service.type "NodePort" }} + nodePort: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.nodePort }} + {{- end }} + port: 8080 + targetPort: http + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + - name: https + {{- if eq .Values.prometheusOperator.admissionWebhooks.deployment.service.type "NodePort"}} + nodePort: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.nodePortTls }} + {{- end }} + port: 443 + targetPort: https + {{- end }} + selector: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + release: {{ $.Release.Name | quote }} + type: "{{ .Values.prometheusOperator.admissionWebhooks.deployment.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml new file mode 100644 index 0000000000..55511da36b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.prometheusOperator.admissionWebhooks.deployment.serviceAccount.automountServiceAccountToken }} +metadata: + name: {{ template "kube-prometheus-stack.operator.admissionWebhooks.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | indent 4 }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml new file mode 100644 index 0000000000..f7543b0f1a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml @@ -0,0 +1,36 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "cilium") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + ## Ensure this is run before the job + helm.sh/hook-weight: "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + endpointSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + {{- if and .Values.prometheusOperator.networkPolicy.cilium .Values.prometheusOperator.networkPolicy.cilium.egress }} + {{ toYaml .Values.prometheusOperator.networkPolicy.cilium.egress | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml new file mode 100644 index 0000000000..4e3b0d9225 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml @@ -0,0 +1,36 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "cilium") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + ## Ensure this is run before the job + helm.sh/hook-weight: "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.patch.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + endpointSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + {{- if and .Values.prometheusOperator.networkPolicy.cilium .Values.prometheusOperator.networkPolicy.cilium.egress }} + {{ toYaml .Values.prometheusOperator.networkPolicy.cilium.egress | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml new file mode 100644 index 0000000000..b81257c168 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +rules: + - apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + - mutatingwebhookconfigurations + verbs: + - get + - update +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") (or .Values.global.cattle.psp.enabled .Values.global.rbac.pspEnabled) }} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} + - apiGroups: ['policy'] +{{- else }} + - apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml new file mode 100644 index 0000000000..4cf1335b22 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml new file mode 100644 index 0000000000..baed83db48 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml @@ -0,0 +1,73 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +{{- with .Values.prometheusOperator.admissionWebhooks.annotations }} +{{ toYaml . | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create +{{- with .Values.prometheusOperator.admissionWebhooks.patch.podAnnotations }} + annotations: +{{ toYaml . | indent 8 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 8 }} + spec: + {{- if .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + {{- end }} + containers: + - name: create + {{- $registry := include "monitoring_registry" . | default .Values.prometheusOperator.admissionWebhooks.patch.image.registry -}} + {{- if .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }}@sha256:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + {{- else }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }} + {{- end }} + imagePullPolicy: {{ .Values.prometheusOperator.admissionWebhooks.patch.image.pullPolicy }} + args: + - create + - --host={{- include "kube-prometheus-stack.operator.admission-webhook.dnsNames" . | replace "\n" "," }} + - --namespace={{ template "kube-prometheus-stack.namespace" . }} + - --secret-name={{ template "kube-prometheus-stack.fullname" . }}-admission + {{- with .Values.prometheusOperator.admissionWebhooks.createSecretJob }} + securityContext: + {{ toYaml .securityContext | nindent 12 }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.resources | indent 12 }} + restartPolicy: OnFailure + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-admission + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.patch.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.patch.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.securityContext | indent 8 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml new file mode 100644 index 0000000000..5639cc9e80 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml @@ -0,0 +1,74 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +{{- with .Values.prometheusOperator.admissionWebhooks.patch.annotations }} +{{ toYaml . | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch +{{- with .Values.prometheusOperator.admissionWebhooks.patch.podAnnotations }} + annotations: +{{ toYaml . | indent 8 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 8 }} + spec: + {{- if .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + {{- end }} + containers: + - name: patch + {{- $registry := include "monitoring_registry" . | default .Values.prometheusOperator.admissionWebhooks.patch.image.registry -}} + {{- if .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }}@sha256:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + {{- else }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }} + {{- end }} + imagePullPolicy: {{ .Values.prometheusOperator.admissionWebhooks.patch.image.pullPolicy }} + args: + - patch + - --webhook-name={{ template "kube-prometheus-stack.fullname" . }}-admission + - --namespace={{ template "kube-prometheus-stack.namespace" . }} + - --secret-name={{ template "kube-prometheus-stack.fullname" . }}-admission + - --patch-failure-policy={{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- with .Values.prometheusOperator.admissionWebhooks.patchWebhookJob }} + securityContext: + {{ toYaml .securityContext | nindent 12 }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.resources | indent 12 }} + restartPolicy: OnFailure + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-admission + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.patch.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.patch.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.securityContext | indent 8 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml new file mode 100644 index 0000000000..864deb52a0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "kubernetes") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + ## Ensure this is run before the job + "helm.sh/hook-weight": "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + podSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + - {} + policyTypes: + - Egress +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml new file mode 100644 index 0000000000..076c467004 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "kubernetes") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + ## Ensure this is run before the job + "helm.sh/hook-weight": "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.patch.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + podSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + - {} + policyTypes: + - Egress +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml new file mode 100644 index 0000000000..0113b6a5d8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml @@ -0,0 +1,47 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +{{- if .Values.global.rbac.pspAnnotations }} +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml new file mode 100644 index 0000000000..f15abf4395 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml new file mode 100644 index 0000000000..30bde920b6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml new file mode 100644 index 0000000000..02594547d1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml new file mode 100644 index 0000000000..da01f3b57e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +webhooks: + - name: prometheusrulemutate.monitoring.coreos.com + {{- if eq .Values.prometheusOperator.admissionWebhooks.failurePolicy "IgnoreOnInstallOnly" }} + failurePolicy: {{ .Release.IsInstall | ternary "Ignore" "Fail" }} + {{- else if .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + failurePolicy: {{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- else if .Values.prometheusOperator.admissionWebhooks.patch.enabled }} + failurePolicy: Ignore + {{- else }} + failurePolicy: Fail + {{- end }} + rules: + - apiGroups: + - monitoring.coreos.com + apiVersions: + - "*" + resources: + - prometheusrules + operations: + - CREATE + - UPDATE + clientConfig: + service: + namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.operator.fullname" $ }}{{ if .Values.prometheusOperator.admissionWebhooks.deployment.enabled }}-webhook{{ end }} + path: /admission-prometheusrules/mutate + {{- if and .Values.prometheusOperator.admissionWebhooks.caBundle (not .Values.prometheusOperator.admissionWebhooks.patch.enabled) (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} + caBundle: {{ .Values.prometheusOperator.admissionWebhooks.caBundle }} + {{- end }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.timeoutSeconds }} + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector }} + namespaceSelector: + {{- with (omit .Values.prometheusOperator.admissionWebhooks.namespaceSelector "matchExpressions") }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions }} + matchExpressions: + {{- with (.Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions) }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- if .Values.prometheusOperator.denyNamespaces }} + - key: kubernetes.io/metadata.name + operator: NotIn + values: + {{- range $namespace := mustUniq .Values.prometheusOperator.denyNamespaces }} + - {{ $namespace }} + {{- end }} + {{- else if and .Values.prometheusOperator.namespaces .Values.prometheusOperator.namespaces.additional }} + - key: kubernetes.io/metadata.name + operator: In + values: + {{- if and .Values.prometheusOperator.namespaces.releaseNamespace (default .Values.prometheusOperator.namespaces.releaseNamespace true) }} + {{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} + - {{ $namespace }} + {{- end }} + {{- range $namespace := mustUniq .Values.prometheusOperator.namespaces.additional }} + - {{ $namespace }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml new file mode 100644 index 0000000000..4827871cca --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +webhooks: + - name: prometheusrulemutate.monitoring.coreos.com + {{- if eq .Values.prometheusOperator.admissionWebhooks.failurePolicy "IgnoreOnInstallOnly" }} + failurePolicy: {{ .Release.IsInstall | ternary "Ignore" "Fail" }} + {{- else if .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + failurePolicy: {{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- else if .Values.prometheusOperator.admissionWebhooks.patch.enabled }} + failurePolicy: Ignore + {{- else }} + failurePolicy: Fail + {{- end }} + rules: + - apiGroups: + - monitoring.coreos.com + apiVersions: + - "*" + resources: + - prometheusrules + operations: + - CREATE + - UPDATE + clientConfig: + service: + namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.operator.fullname" $ }}{{ if .Values.prometheusOperator.admissionWebhooks.deployment.enabled }}-webhook{{ end }} + path: /admission-prometheusrules/validate + {{- if and .Values.prometheusOperator.admissionWebhooks.caBundle (not .Values.prometheusOperator.admissionWebhooks.patch.enabled) (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} + caBundle: {{ .Values.prometheusOperator.admissionWebhooks.caBundle }} + {{- end }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.timeoutSeconds }} + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector }} + namespaceSelector: + {{- with (omit .Values.prometheusOperator.admissionWebhooks.namespaceSelector "matchExpressions") }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions }} + matchExpressions: + {{- with (.Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions) }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.prometheusOperator.denyNamespaces }} + - key: kubernetes.io/metadata.name + operator: NotIn + values: + {{- range $namespace := mustUniq .Values.prometheusOperator.denyNamespaces }} + - {{ $namespace }} + {{- end }} + {{- else if and .Values.prometheusOperator.namespaces .Values.prometheusOperator.namespaces.additional }} + - key: kubernetes.io/metadata.name + operator: In + values: + {{- if and .Values.prometheusOperator.namespaces.releaseNamespace (default .Values.prometheusOperator.namespaces.releaseNamespace true) }} + {{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} + - {{ $namespace }} + {{- end }} + {{- range $namespace := mustUniq .Values.prometheusOperator.namespaces.additional }} + - {{ $namespace }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/certmanager.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/certmanager.yaml new file mode 100644 index 0000000000..cb27e49f48 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/certmanager.yaml @@ -0,0 +1,55 @@ +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled -}} +{{- if not .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef -}} +# Create a selfsigned Issuer, in order to create a root CA certificate for +# signing webhook serving certificates +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-self-signed-issuer + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + selfSigned: {} +--- +# Generate a CA Certificate used to sign certificates for the webhook +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-root-cert + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-root-cert + duration: {{ .Values.prometheusOperator.admissionWebhooks.certManager.rootCert.duration | default "43800h0m0s" | quote }} + issuerRef: + name: {{ template "kube-prometheus-stack.fullname" . }}-self-signed-issuer + commonName: "ca.webhook.kube-prometheus-stack" + isCA: true +--- +# Create an Issuer that uses the above generated CA certificate to issue certs +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-root-issuer + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + ca: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-root-cert +{{- end }} +--- +# generate a server certificate for the apiservices to use +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission + duration: {{ .Values.prometheusOperator.admissionWebhooks.certManager.admissionCert.duration | default "8760h0m0s" | quote }} + issuerRef: + {{- if .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef }} + {{- toYaml .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef | nindent 4 }} + {{- else }} + name: {{ template "kube-prometheus-stack.fullname" . }}-root-issuer + {{- end }} + dnsNames: + {{- include "kube-prometheus-stack.operator.admission-webhook.dnsNames" . | splitList "\n" | toYaml | nindent 4 }} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/ciliumnetworkpolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/ciliumnetworkpolicy.yaml new file mode 100644 index 0000000000..07e2e99967 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/ciliumnetworkpolicy.yaml @@ -0,0 +1,40 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "cilium") }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +spec: + endpointSelector: + matchLabels: + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + app: {{ template "kube-prometheus-stack.name" . }}-operator + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator.labels" $ | nindent 6 }} + {{- end }} + egress: + {{- if and .Values.prometheusOperator.networkPolicy.cilium .Values.prometheusOperator.networkPolicy.cilium.egress }} + {{ toYaml .Values.prometheusOperator.networkPolicy.cilium.egress | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} + ingress: + - toPorts: + - ports: + {{- if .Values.prometheusOperator.tls.enabled }} + - port: {{ .Values.prometheusOperator.tls.internalPort | quote }} + {{- else }} + - port: "8080" + {{- end }} + protocol: "TCP" + {{- if not .Values.prometheusOperator.tls.enabled }} + rules: + http: + - method: "GET" + path: "/metrics" + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/clusterrole.yaml new file mode 100644 index 0000000000..fd11b69eed --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/clusterrole.yaml @@ -0,0 +1,109 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - alertmanagers/finalizers + - alertmanagers/status + - alertmanagerconfigs + - prometheuses + - prometheuses/finalizers + - prometheuses/status + - prometheusagents + - prometheusagents/finalizers + - prometheusagents/status + - thanosrulers + - thanosrulers/finalizers + - thanosrulers/status + - scrapeconfigs + - servicemonitors + - podmonitors + - probes + - prometheusrules + verbs: + - '*' +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - '*' +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +- apiGroups: + - "" + resources: + - pods + verbs: + - list + - delete +- apiGroups: + - "" + resources: + - services + - services/finalizers + - endpoints + verbs: + - get + - create + - update + - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - patch + - create +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get +{{- if .Capabilities.APIVersions.Has "discovery.k8s.io/v1/EndpointSlice" }} +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/clusterrolebinding.yaml new file mode 100644 index 0000000000..ad9e3ef6c5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.operator.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/deployment.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/deployment.yaml new file mode 100644 index 0000000000..8a01b2912a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/deployment.yaml @@ -0,0 +1,204 @@ +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +{{- $defaultKubeletSvcName := printf "%s-kubelet" (include "kube-prometheus-stack.fullname" .) }} +{{- if .Values.prometheusOperator.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.labels }} +{{ toYaml .Values.prometheusOperator.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.annotations | indent 4 }} +{{- end }} +spec: + replicas: 1 + revisionHistoryLimit: {{ .Values.prometheusOperator.revisionHistoryLimit }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + {{- with .Values.prometheusOperator.strategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 8 }} +{{- if .Values.prometheusOperator.podLabels }} +{{ toYaml .Values.prometheusOperator.podLabels | indent 8 }} +{{- end }} +{{- if .Values.prometheusOperator.podAnnotations }} + annotations: +{{ toYaml .Values.prometheusOperator.podAnnotations | indent 8 }} +{{- end }} + spec: + {{- if .Values.prometheusOperator.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.priorityClassName }} + {{- end }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "kube-prometheus-stack.imagePullSecrets" . | indent 8 }} + {{- end }} + containers: + - name: {{ template "kube-prometheus-stack.name" . }} + {{- $base_registry := (include "monitoring_registry" .) }} + {{- $configReloaderRegistry := $base_registry | default .Values.prometheusOperator.prometheusConfigReloader.image.registry -}} + {{- $operatorRegistry := $base_registry | default .Values.prometheusOperator.image.registry -}} + {{- $thanosRegistry := $base_registry | default .Values.prometheusOperator.thanosImage.registry -}} + {{- if .Values.prometheusOperator.image.sha }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.image.repository }}:{{ .Values.prometheusOperator.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.prometheusOperator.image.sha }}" + {{- else }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.image.repository }}:{{ .Values.prometheusOperator.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: "{{ .Values.prometheusOperator.image.pullPolicy }}" + args: + {{- if .Values.prometheusOperator.kubeletService.enabled }} + - --kubelet-service={{ .Values.prometheusOperator.kubeletService.namespace }}/{{ default $defaultKubeletSvcName .Values.prometheusOperator.kubeletService.name }} + {{- end }} + {{- if .Values.prometheusOperator.logFormat }} + - --log-format={{ .Values.prometheusOperator.logFormat }} + {{- end }} + {{- if .Values.prometheusOperator.logLevel }} + - --log-level={{ .Values.prometheusOperator.logLevel }} + {{- end }} + {{- if .Values.prometheusOperator.denyNamespaces }} + - --deny-namespaces={{ tpl (.Values.prometheusOperator.denyNamespaces | join ",") $ }} + {{- end }} + {{- with $.Values.prometheusOperator.namespaces }} + {{- $namespaces := list }} + {{- if .releaseNamespace }} + {{- $namespaces = append $namespaces $namespace }} + {{- end }} + {{- if .additional }} + {{- range $ns := .additional }} + {{- $namespaces = append $namespaces (tpl $ns $) }} + {{- end }} + {{- end }} + - --namespaces={{ $namespaces | mustUniq | join "," }} + {{- end }} + - --localhost=127.0.0.1 + {{- if .Values.prometheusOperator.prometheusDefaultBaseImage }} + - --prometheus-default-base-image={{ $base_registry | default .Values.prometheusOperator.prometheusDefaultBaseImageRegistry }}/{{ .Values.prometheusOperator.prometheusDefaultBaseImage }} + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerDefaultBaseImage }} + - --alertmanager-default-base-image={{ $base_registry | default .Values.prometheusOperator.alertmanagerDefaultBaseImageRegistry }}/{{ .Values.prometheusOperator.alertmanagerDefaultBaseImage }} + {{- end }} + {{- if .Values.prometheusOperator.prometheusConfigReloader.image.sha }} + - --prometheus-config-reloader={{ $configReloaderRegistry }}/{{ .Values.prometheusOperator.prometheusConfigReloader.image.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloader.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.prometheusOperator.prometheusConfigReloader.image.sha }} + {{- else }} + - --prometheus-config-reloader={{ $configReloaderRegistry }}/{{ .Values.prometheusOperator.prometheusConfigReloader.image.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloader.image.tag | default .Chart.AppVersion }} + {{- end }} + - --config-reloader-cpu-request={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).requests).cpu) | default 0 }} + - --config-reloader-cpu-limit={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).limits).cpu) | default 0 }} + - --config-reloader-memory-request={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).requests).memory) | default 0 }} + - --config-reloader-memory-limit={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).limits).memory) | default 0 }} + {{- if .Values.prometheusOperator.prometheusConfigReloader.enableProbe }} + - --enable-config-reloader-probes=true + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerInstanceNamespaces }} + - --alertmanager-instance-namespaces={{ .Values.prometheusOperator.alertmanagerInstanceNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerInstanceSelector }} + - --alertmanager-instance-selector={{ .Values.prometheusOperator.alertmanagerInstanceSelector }} + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerConfigNamespaces }} + - --alertmanager-config-namespaces={{ .Values.prometheusOperator.alertmanagerConfigNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.prometheusInstanceNamespaces }} + - --prometheus-instance-namespaces={{ .Values.prometheusOperator.prometheusInstanceNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.prometheusInstanceSelector }} + - --prometheus-instance-selector={{ .Values.prometheusOperator.prometheusInstanceSelector }} + {{- end }} + {{- if .Values.prometheusOperator.thanosImage.sha }} + - --thanos-default-base-image={{ $thanosRegistry }}/{{ .Values.prometheusOperator.thanosImage.repository }}:{{ .Values.prometheusOperator.thanosImage.tag }}@sha256:{{ .Values.prometheusOperator.thanosImage.sha }} + {{- else }} + - --thanos-default-base-image={{ $thanosRegistry }}/{{ .Values.prometheusOperator.thanosImage.repository }}:{{ .Values.prometheusOperator.thanosImage.tag }} + {{- end }} + {{- if .Values.prometheusOperator.thanosRulerInstanceNamespaces }} + - --thanos-ruler-instance-namespaces={{ .Values.prometheusOperator.thanosRulerInstanceNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.thanosRulerInstanceSelector }} + - --thanos-ruler-instance-selector={{ .Values.prometheusOperator.thanosRulerInstanceSelector }} + {{- end }} + {{- if .Values.prometheusOperator.secretFieldSelector }} + - --secret-field-selector={{ tpl (.Values.prometheusOperator.secretFieldSelector) $ }} + {{- end }} + {{- if .Values.prometheusOperator.clusterDomain }} + - --cluster-domain={{ .Values.prometheusOperator.clusterDomain }} + {{- end }} + {{- if .Values.prometheusOperator.tls.enabled }} + - --web.enable-tls=true + - --web.cert-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.crt{{ else }}cert{{ end }} + - --web.key-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.key{{ else }}key{{ end }} + - --web.listen-address=:{{ .Values.prometheusOperator.tls.internalPort }} + - --web.tls-min-version={{ .Values.prometheusOperator.tls.tlsMinVersion }} + ports: + - containerPort: {{ .Values.prometheusOperator.tls.internalPort }} + name: https + {{- else }} + ports: + - containerPort: 8080 + name: http + {{- end }} + env: + {{- range $key, $value := .Values.prometheusOperator.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.resources | indent 12 }} + securityContext: +{{ toYaml .Values.prometheusOperator.containerSecurityContext | indent 12 }} + volumeMounts: + {{- if .Values.prometheusOperator.tls.enabled }} + - name: tls-secret + mountPath: /cert + readOnly: true + {{- end }} + {{- with .Values.prometheusOperator.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + {{- if .Values.prometheusOperator.tls.enabled }} + - name: tls-secret + secret: + defaultMode: 420 + secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission + {{- end }} + {{- with .Values.prometheusOperator.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheusOperator.dnsConfig }} + dnsConfig: +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.securityContext | indent 8 }} +{{- end }} + serviceAccountName: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.prometheusOperator.automountServiceAccountToken }} +{{- if .Values.prometheusOperator.hostNetwork }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.tolerations }} +{{ toYaml . | indent 8 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/networkpolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/networkpolicy.yaml new file mode 100644 index 0000000000..cfd5b0b8c7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/networkpolicy.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "kubernetes") }} +apiVersion: {{ template "kube-prometheus-stack.prometheus.networkPolicy.apiVersion" . }} +kind: NetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +spec: + egress: + - {} + ingress: + - ports: + {{- if .Values.prometheusOperator.tls.enabled }} + - port: {{ .Values.prometheusOperator.tls.internalPort }} + {{- else }} + - port: 8080 + {{- end }} + policyTypes: + - Egress + - Ingress + podSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/psp-clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/psp-clusterrole.yaml new file mode 100644 index 0000000000..61bc3d9040 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/psp-clusterrole.yaml @@ -0,0 +1,21 @@ +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +{{- if and .Values.prometheusOperator.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-psp + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.operator.fullname" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..40e0fc5c15 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/psp-clusterrolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.prometheusOperator.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-psp + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-psp +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/psp.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/psp.yaml new file mode 100644 index 0000000000..28a9075d3e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/psp.yaml @@ -0,0 +1,46 @@ +{{- if and .Values.prometheusOperator.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: {{ .Values.prometheusOperator.hostNetwork }} + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/service.yaml new file mode 100644 index 0000000000..d45ab22d08 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/service.yaml @@ -0,0 +1,57 @@ +{{- if .Values.prometheusOperator.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.service.labels }} +{{ toYaml .Values.prometheusOperator.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.service.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheusOperator.service.clusterIP }} + clusterIP: {{ .Values.prometheusOperator.service.clusterIP }} +{{- end }} +{{- if .Values.prometheusOperator.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheusOperator.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheusOperator.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheusOperator.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheusOperator.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheusOperator.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheusOperator.service.externalTrafficPolicy }} +{{- end }} + ports: + {{- if not .Values.prometheusOperator.tls.enabled }} + - name: http + {{- if eq .Values.prometheusOperator.service.type "NodePort" }} + nodePort: {{ .Values.prometheusOperator.service.nodePort }} + {{- end }} + port: 8080 + targetPort: http + {{- end }} + {{- if .Values.prometheusOperator.tls.enabled }} + - name: https + {{- if eq .Values.prometheusOperator.service.type "NodePort"}} + nodePort: {{ .Values.prometheusOperator.service.nodePortTls }} + {{- end }} + port: 443 + targetPort: https + {{- end }} + selector: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + type: "{{ .Values.prometheusOperator.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/serviceaccount.yaml new file mode 100644 index 0000000000..4f84974f9b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/serviceaccount.yaml @@ -0,0 +1,14 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +automountServiceAccountToken: {{ .Values.prometheusOperator.serviceAccount.automountServiceAccountToken }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/servicemonitor.yaml new file mode 100644 index 0000000000..cbe79e1253 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/servicemonitor.yaml @@ -0,0 +1,57 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- with .Values.prometheusOperator.serviceMonitor.additionalLabels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.prometheusOperator.serviceMonitor | nindent 2 }} + endpoints: + {{- if .Values.prometheusOperator.tls.enabled }} + - port: https + scheme: https + tlsConfig: + serverName: {{ template "kube-prometheus-stack.operator.fullname" . }} + ca: + secret: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + key: {{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}ca.crt{{ else }}ca{{ end }} + optional: false + {{- else }} + - port: http + {{- end }} + honorLabels: true + {{- if .Values.prometheusOperator.serviceMonitor.interval }} + interval: {{ .Values.prometheusOperator.serviceMonitor.interval }} + {{- end }} + metricRelabelings: + {{- if .Values.prometheusOperator.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.prometheusOperator.serviceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.prometheusOperator.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.prometheusOperator.serviceMonitor.relabelings | indent 6 }} +{{- end }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/verticalpodautoscaler.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/verticalpodautoscaler.yaml new file mode 100644 index 0000000000..f225d16dde --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus-operator/verticalpodautoscaler.yaml @@ -0,0 +1,40 @@ +{{- if and (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1") (.Values.prometheusOperator.verticalPodAutoscaler.enabled) }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +spec: + {{- with .Values.prometheusOperator.verticalPodAutoscaler.recommenders }} + recommenders: + {{- toYaml . | nindent 4 }} + {{- end }} + resourcePolicy: + containerPolicies: + - containerName: {{ template "kube-prometheus-stack.name" . }} + {{- with .Values.prometheusOperator.verticalPodAutoscaler.controlledResources }} + controlledResources: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.prometheusOperator.verticalPodAutoscaler.controlledValues }} + controlledValues: {{ .Values.prometheusOperator.verticalPodAutoscaler.controlledValues }} + {{- end }} + {{- if .Values.prometheusOperator.verticalPodAutoscaler.maxAllowed }} + maxAllowed: + {{- toYaml .Values.prometheusOperator.verticalPodAutoscaler.maxAllowed | nindent 8 }} + {{- end }} + {{- if .Values.prometheusOperator.verticalPodAutoscaler.minAllowed }} + minAllowed: + {{- toYaml .Values.prometheusOperator.verticalPodAutoscaler.minAllowed | nindent 8 }} + {{- end }} + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + {{- with .Values.prometheusOperator.verticalPodAutoscaler.updatePolicy }} + updatePolicy: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/_rules.tpl b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/_rules.tpl new file mode 100644 index 0000000000..4a8213d089 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/_rules.tpl @@ -0,0 +1,44 @@ +{{- /* +Generated file. Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- define "rules.names" }} +rules: + - "alertmanager.rules" + - "config-reloaders" + - "etcd" + - "general.rules" + - "k8s.rules.container-cpu-usage-seconds-total" + - "k8s.rules.container-memory-cache" + - "k8s.rules.container-memory-rss" + - "k8s.rules.container-memory-swap" + - "k8s.rules.container-memory-working-set-bytes" + - "k8s.rules.container-resource" + - "k8s.rules.pod-owner" + - "kube-apiserver-availability.rules" + - "kube-apiserver-burnrate.rules" + - "kube-apiserver-histogram.rules" + - "kube-apiserver-slos" + - "kube-prometheus-general.rules" + - "kube-prometheus-node-recording.rules" + - "kube-scheduler.rules" + - "kube-state-metrics" + - "kubelet.rules" + - "kubernetes-apps" + - "kubernetes-resources" + - "kubernetes-storage" + - "kubernetes-system" + - "kubernetes-system-kube-proxy" + - "kubernetes-system-apiserver" + - "kubernetes-system-kubelet" + - "kubernetes-system-controller-manager" + - "kubernetes-system-scheduler" + - "node-exporter.rules" + - "node-exporter" + - "node.rules" + - "node-network" + - "prometheus-operator" + - "prometheus" + - "windows.node.rules" + - "windows.pod.rules" +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalAlertRelabelConfigs.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalAlertRelabelConfigs.yaml new file mode 100644 index 0000000000..bff930981a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalAlertRelabelConfigs.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-relabel-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-am-relabel-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + additional-alert-relabel-configs.yaml: {{ toYaml .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs | b64enc | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalAlertmanagerConfigs.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalAlertmanagerConfigs.yaml new file mode 100644 index 0000000000..2fe8fdb816 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalAlertmanagerConfigs.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-am-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + additional-alertmanager-configs.yaml: {{ tpl (toYaml .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs) . | b64enc | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalPrometheusRules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalPrometheusRules.yaml new file mode 100644 index 0000000000..cb4aabaa7b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalPrometheusRules.yaml @@ -0,0 +1,43 @@ +{{- if or .Values.additionalPrometheusRules .Values.additionalPrometheusRulesMap}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-additional-prometheus-rules + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- if .Values.additionalPrometheusRulesMap }} +{{- range $prometheusRuleName, $prometheusRule := .Values.additionalPrometheusRulesMap }} + - apiVersion: monitoring.coreos.com/v1 + kind: PrometheusRule + metadata: + name: {{ template "kube-prometheus-stack.name" $ }}-{{ $prometheusRuleName }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }} +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $prometheusRule.additionalLabels }} +{{ toYaml $prometheusRule.additionalLabels | indent 8 }} + {{- end }} + spec: + groups: +{{ toYaml $prometheusRule.groups| indent 8 }} +{{- end }} +{{- else }} +{{- range .Values.additionalPrometheusRules }} + - apiVersion: monitoring.coreos.com/v1 + kind: PrometheusRule + metadata: + name: {{ template "kube-prometheus-stack.name" $ }}-{{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }} +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + groups: +{{ toYaml .groups| indent 8 }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalScrapeConfigs.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalScrapeConfigs.yaml new file mode 100644 index 0000000000..ebdf766fde --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/additionalScrapeConfigs.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalScrapeConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-scrape-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-scrape-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- if eq ( typeOf .Values.prometheus.prometheusSpec.additionalScrapeConfigs ) "string" }} + additional-scrape-configs.yaml: {{ tpl .Values.prometheus.prometheusSpec.additionalScrapeConfigs $ | b64enc | quote }} +{{- else }} + additional-scrape-configs.yaml: {{ tpl (toYaml .Values.prometheus.prometheusSpec.additionalScrapeConfigs) $ | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ciliumnetworkpolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ciliumnetworkpolicy.yaml new file mode 100644 index 0000000000..74d61d7c13 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ciliumnetworkpolicy.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.prometheus.networkPolicy.enabled (eq .Values.prometheus.networkPolicy.flavor "cilium") }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + {{- include "kube-prometheus-stack.labels" . | nindent 4 }} +spec: + endpointSelector: + {{- if .Values.prometheus.networkPolicy.cilium.endpointSelector }} + {{- toYaml .Values.prometheus.networkPolicy.cilium.endpointSelector | nindent 4 }} + {{- else }} + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.prometheus.crname" . }}]} + {{- end }} + {{- if and .Values.prometheus.networkPolicy.cilium .Values.prometheus.networkPolicy.cilium.egress }} + egress: + {{ toYaml .Values.prometheus.networkPolicy.cilium.egress | nindent 4 }} + {{- end }} + {{- if and .Values.prometheus.networkPolicy.cilium .Values.prometheus.networkPolicy.cilium.ingress }} + ingress: + {{ toYaml .Values.prometheus.networkPolicy.cilium.ingress | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/clusterrole.yaml new file mode 100644 index 0000000000..3585b5db11 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/clusterrole.yaml @@ -0,0 +1,30 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +# This permission are not in the kube-prometheus repo +# they're grabbed from https://github.com/prometheus/prometheus/blob/master/documentation/examples/rbac-setup.yml +- apiGroups: [""] + resources: + - nodes + - nodes/metrics + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: + - "networking.k8s.io" + resources: + - ingresses + verbs: ["get", "list", "watch"] +- nonResourceURLs: ["/metrics", "/metrics/cadvisor"] + verbs: ["get"] +{{- if .Values.prometheus.additionalRulesForClusterRole }} +{{ toYaml .Values.prometheus.additionalRulesForClusterRole | indent 0 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/clusterrolebinding.yaml new file mode 100644 index 0000000000..9fc4f65da4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/clusterrolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} + diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/csi-secret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/csi-secret.yaml new file mode 100644 index 0000000000..e05382f633 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/csi-secret.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.prometheus.prometheusSpec.thanos .Values.prometheus.prometheusSpec.thanos.secretProviderClass }} +--- +apiVersion: secrets-store.csi.x-k8s.io/v1alpha1 +kind: SecretProviderClass +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +spec: +{{ toYaml .Values.prometheus.prometheusSpec.thanos.secretProviderClass | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/extrasecret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/extrasecret.yaml new file mode 100644 index 0000000000..17f3478a46 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.prometheus.extraSecret.data -}} +{{- $secretName := printf "prometheus-%s-extra" (include "kube-prometheus-stack.fullname" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.prometheus.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.extraSecret.annotations }} + annotations: +{{ toYaml .Values.prometheus.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.prometheus.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ingress.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ingress.yaml new file mode 100644 index 0000000000..d2f6af5dd1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ingress.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.ingress.enabled -}} + {{- $pathType := .Values.prometheus.ingress.pathType | default "ImplementationSpecific" -}} + {{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" -}} + {{- $servicePort := .Values.prometheus.ingress.servicePort | default .Values.prometheus.service.port -}} + {{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix -}} + {{- $paths := .Values.prometheus.ingress.paths | default $routePrefix -}} + {{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} + {{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: +{{- if .Values.prometheus.ingress.annotations }} + annotations: + {{- tpl (toYaml .Values.prometheus.ingress.annotations) . | nindent 4 }} +{{- end }} + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.ingress.labels }} +{{ toYaml .Values.prometheus.ingress.labels | indent 4 }} +{{- end }} +spec: + {{- if $apiIsStable }} + {{- if .Values.prometheus.ingress.ingressClassName }} + ingressClassName: {{ .Values.prometheus.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.prometheus.ingress.hosts }} + {{- range $host := .Values.prometheus.ingress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.prometheus.ingress.tls }} + tls: +{{ tpl (toYaml .Values.prometheus.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ingressThanosSidecar.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ingressThanosSidecar.yaml new file mode 100644 index 0000000000..3f507cfa9f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ingressThanosSidecar.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosIngress.enabled }} +{{- $pathType := .Values.prometheus.thanosIngress.pathType | default "" }} +{{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "thanos-discovery" }} +{{- $thanosPort := .Values.prometheus.thanosIngress.servicePort -}} +{{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix }} +{{- $paths := .Values.prometheus.thanosIngress.paths | default $routePrefix -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: +{{- if .Values.prometheus.thanosIngress.annotations }} + annotations: + {{- tpl (toYaml .Values.prometheus.thanosIngress.annotations) . | nindent 4 }} +{{- end }} + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-gateway + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosIngress.labels }} +{{ toYaml .Values.prometheus.thanosIngress.labels | indent 4 }} +{{- end }} +spec: + {{- if $apiIsStable }} + {{- if .Values.prometheus.thanosIngress.ingressClassName }} + ingressClassName: {{ .Values.prometheus.thanosIngress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.prometheus.thanosIngress.hosts }} + {{- range $host := .Values.prometheus.thanosIngress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $thanosPort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $thanosPort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $thanosPort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $thanosPort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.prometheus.thanosIngress.tls }} + tls: +{{ tpl (toYaml .Values.prometheus.thanosIngress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ingressperreplica.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ingressperreplica.yaml new file mode 100644 index 0000000000..1d76d135c8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/ingressperreplica.yaml @@ -0,0 +1,67 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.servicePerReplica.enabled .Values.prometheus.ingressPerReplica.enabled }} +{{- $pathType := .Values.prometheus.ingressPerReplica.pathType | default "" }} +{{- $count := .Values.prometheus.prometheusSpec.replicas | int -}} +{{- $servicePort := .Values.prometheus.servicePerReplica.port -}} +{{- $ingressValues := .Values.prometheus.ingressPerReplica -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-ingressperreplica + namespace: {{ template "kube-prometheus-stack.namespace" $ }} +items: +{{ range $i, $e := until $count }} + - kind: Ingress + apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" $ }} + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-prometheus + {{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $ingressValues.labels }} +{{ toYaml $ingressValues.labels | indent 8 }} + {{- end }} + {{- if $ingressValues.annotations }} + annotations: + {{- tpl (toYaml $ingressValues.annotations) $ | nindent 8 }} + {{- end }} + spec: + {{- if $apiIsStable }} + {{- if $ingressValues.ingressClassName }} + ingressClassName: {{ $ingressValues.ingressClassName }} + {{- end }} + {{- end }} + rules: + - host: {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + http: + paths: + {{- range $p := $ingressValues.paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- if or $ingressValues.tlsSecretName $ingressValues.tlsSecretPerReplica.enabled }} + tls: + - hosts: + - {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + {{- if $ingressValues.tlsSecretPerReplica.enabled }} + secretName: {{ $ingressValues.tlsSecretPerReplica.prefix }}-{{ $i }} + {{- else }} + secretName: {{ $ingressValues.tlsSecretName }} + {{- end }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/networkpolicy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/networkpolicy.yaml new file mode 100644 index 0000000000..1296a79063 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/networkpolicy.yaml @@ -0,0 +1,34 @@ +{{- if and .Values.prometheus.networkPolicy.enabled (eq .Values.prometheus.networkPolicy.flavor "kubernetes") }} +apiVersion: {{ template "kube-prometheus-stack.prometheus.networkPolicy.apiVersion" . }} +kind: NetworkPolicy +metadata: + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + {{- include "kube-prometheus-stack.labels" . | nindent 4 }} + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + {{- if .Values.prometheus.networkPolicy.egress }} + egress: + {{- toYaml .Values.prometheus.networkPolicy.egress | nindent 4 }} + {{- end }} + {{- if .Values.prometheus.networkPolicy.ingress }} + ingress: + {{- toYaml .Values.prometheus.networkPolicy.ingress | nindent 4 }} + {{- end }} + policyTypes: + - Egress + - Ingress + podSelector: + {{- if .Values.prometheus.networkPolicy.podSelector }} + {{- toYaml .Values.prometheus.networkPolicy.podSelector | nindent 4 }} + {{- else }} + matchLabels: + {{- if .Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + {{- else }} + app.kubernetes.io/name: prometheus + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/nginx-config.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/nginx-config.yaml new file mode 100644 index 0000000000..e4d91f9a9e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/nginx-config.yaml @@ -0,0 +1,68 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-nginx-proxy-config + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.annotations }} + annotations: +{{ toYaml .Values.prometheus.annotations | indent 4 }} +{{- end }} +data: + nginx.conf: |- + worker_processes auto; + error_log /dev/stdout warn; + pid /var/cache/nginx/nginx.pid; + + events { + worker_connections 1024; + } + + http { + include /etc/nginx/mime.types; + log_format main '[$time_local - $status] $remote_addr - $remote_user $request ($http_referer)'; + + proxy_connect_timeout 10; + proxy_read_timeout 180; + proxy_send_timeout 5; + proxy_buffering off; + proxy_cache_path /var/cache/nginx/cache levels=1:2 keys_zone=my_zone:100m inactive=1d max_size=10g; + + server { + listen 8081; + access_log off; + + gzip on; + gzip_min_length 1k; + gzip_comp_level 2; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript image/jpeg image/gif image/png; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + proxy_set_header Host $host; + + location / { + proxy_cache my_zone; + proxy_cache_valid 200 302 1d; + proxy_cache_valid 301 30d; + proxy_cache_valid any 5m; + proxy_cache_bypass $http_cache_control; + add_header X-Proxy-Cache $upstream_cache_status; + add_header Cache-Control "public"; + + proxy_pass http://localhost:9090/; + + sub_filter_once off; + sub_filter 'var PATH_PREFIX = "";' 'var PATH_PREFIX = ".";'; + + if ($request_filename ~ .*\.(?:js|css|jpg|jpeg|gif|png|ico|cur|gz|svg|svgz|mp4|ogg|ogv|webm)$) { + expires 90d; + } + + rewrite ^/k8s/clusters/.*/proxy(.*) /$1 break; + + } + } + } diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/podDisruptionBudget.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/podDisruptionBudget.yaml new file mode 100644 index 0000000000..48f3f1f5a6 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/podDisruptionBudget.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.podDisruptionBudget.enabled }} +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.prometheus.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.prometheus.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.prometheus.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.prometheus.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + {{- if .Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + {{- else }} + app.kubernetes.io/name: prometheus + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/podmonitors.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/podmonitors.yaml new file mode 100644 index 0000000000..4e748c23b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/podmonitors.yaml @@ -0,0 +1,38 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.additionalPodMonitors }} +apiVersion: v1 +kind: List +items: +{{- range .Values.prometheus.additionalPodMonitors }} + - apiVersion: monitoring.coreos.com/v1 + kind: PodMonitor + metadata: + name: {{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + {{- include "servicemonitor.scrapeLimits" . | nindent 6 }} + podMetricsEndpoints: +{{ toYaml .podMetricsEndpoints | indent 8 }} + {{- if .jobLabel }} + jobLabel: {{ .jobLabel }} + {{- end }} + {{- if .namespaceSelector }} + namespaceSelector: +{{ toYaml .namespaceSelector | indent 8 }} + {{- end }} + selector: +{{ toYaml .selector | indent 8 }} + {{- if .podTargetLabels }} + podTargetLabels: +{{ toYaml .podTargetLabels | indent 8 }} + {{- end }} + {{- if .sampleLimit }} + sampleLimit: {{ .sampleLimit }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/prometheus.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/prometheus.yaml new file mode 100644 index 0000000000..5c3c8d4d1f --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/prometheus.yaml @@ -0,0 +1,472 @@ +{{- if .Values.prometheus.enabled }} +{{- if .Values.prometheus.agentMode }} +apiVersion: monitoring.coreos.com/v1alpha1 +kind: PrometheusAgent +{{- else }} +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +{{- end }} +metadata: + name: {{ template "kube-prometheus-stack.prometheus.crname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.annotations }} + annotations: +{{ toYaml .Values.prometheus.annotations | indent 4 }} +{{- end }} +spec: +{{- if and (not .Values.prometheus.agentMode) (or .Values.prometheus.prometheusSpec.alertingEndpoints .Values.alertmanager.enabled) }} + alerting: + alertmanagers: +{{- if .Values.prometheus.prometheusSpec.alertingEndpoints }} +{{ toYaml .Values.prometheus.prometheusSpec.alertingEndpoints | indent 6 }} +{{- else if .Values.alertmanager.enabled }} + - namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + port: {{ .Values.alertmanager.alertmanagerSpec.portName }} + {{- if .Values.alertmanager.alertmanagerSpec.routePrefix }} + pathPrefix: "{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" + {{- end }} + {{- if .Values.alertmanager.alertmanagerSpec.scheme }} + scheme: {{ .Values.alertmanager.alertmanagerSpec.scheme }} + {{- end }} + {{- if .Values.alertmanager.alertmanagerSpec.tlsConfig }} + tlsConfig: +{{ toYaml .Values.alertmanager.alertmanagerSpec.tlsConfig | indent 10 }} + {{- end }} + apiVersion: {{ .Values.alertmanager.apiVersion }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.apiserverConfig }} + apiserverConfig: +{{ toYaml .Values.prometheus.prometheusSpec.apiserverConfig | indent 4}} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.image }} + {{- $registry := include "monitoring_registry" . | default .Values.prometheus.prometheusSpec.image.registry -}} + {{- if and .Values.prometheus.prometheusSpec.image.tag .Values.prometheus.prometheusSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}:{{ .Values.prometheus.prometheusSpec.image.tag }}@sha256:{{ .Values.prometheus.prometheusSpec.image.sha }}" + {{- else if .Values.prometheus.prometheusSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}@sha256:{{ .Values.prometheus.prometheusSpec.image.sha }}" + {{- else if .Values.prometheus.prometheusSpec.image.tag }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}:{{ .Values.prometheus.prometheusSpec.image.tag }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}" + {{- end }} + version: {{ default .Values.prometheus.prometheusSpec.image.tag .Values.prometheus.prometheusSpec.version }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalArgs }} + additionalArgs: +{{ toYaml .Values.prometheus.prometheusSpec.additionalArgs | indent 4}} +{{- end -}} +{{- if .Values.prometheus.prometheusSpec.externalLabels }} + externalLabels: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.externalLabels | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.prometheusExternalLabelNameClear }} + prometheusExternalLabelName: "" +{{- else if .Values.prometheus.prometheusSpec.prometheusExternalLabelName }} + prometheusExternalLabelName: "{{ .Values.prometheus.prometheusSpec.prometheusExternalLabelName }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.replicaExternalLabelNameClear }} + replicaExternalLabelName: "" +{{- else if .Values.prometheus.prometheusSpec.replicaExternalLabelName }} + replicaExternalLabelName: "{{ .Values.prometheus.prometheusSpec.replicaExternalLabelName }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enableRemoteWriteReceiver }} + enableRemoteWriteReceiver: {{ .Values.prometheus.prometheusSpec.enableRemoteWriteReceiver }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.externalUrl }} + externalUrl: "{{ tpl .Values.prometheus.prometheusSpec.externalUrl . }}" +{{- else if and .Values.prometheus.ingress.enabled .Values.prometheus.ingress.hosts }} + externalUrl: "http://{{ tpl (index .Values.prometheus.ingress.hosts 0) . }}{{ .Values.prometheus.prometheusSpec.routePrefix }}" +{{- else if not (or (kindIs "invalid" .Values.global.cattle.url) (kindIs "invalid" .Values.global.cattle.clusterId)) }} + externalUrl: "{{ .Values.global.cattle.url }}/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ template "kube-prometheus-stack.namespace" . }}/services/http:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}/proxy" +{{- else }} + externalUrl: http://{{ template "kube-prometheus-stack.fullname" . }}-prometheus.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.prometheus.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.prometheus.prometheusSpec.nodeSelector }} +{{ toYaml .Values.prometheus.prometheusSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.prometheus.prometheusSpec.paused }} + replicas: {{ .Values.prometheus.prometheusSpec.replicas }} + shards: {{ .Values.prometheus.prometheusSpec.shards }} + logLevel: {{ .Values.prometheus.prometheusSpec.logLevel }} + logFormat: {{ .Values.prometheus.prometheusSpec.logFormat }} + listenLocal: {{ .Values.prometheus.prometheusSpec.listenLocal }} +{{- if not .Values.prometheus.agentMode }} + enableAdminAPI: {{ .Values.prometheus.prometheusSpec.enableAdminAPI }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.web }} + web: +{{ toYaml .Values.prometheus.prometheusSpec.web | indent 4 }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.exemplars }} + exemplars: + {{ toYaml .Values.prometheus.prometheusSpec.exemplars | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enableFeatures }} + enableFeatures: +{{- range $enableFeatures := .Values.prometheus.prometheusSpec.enableFeatures }} + - {{ tpl $enableFeatures $ }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeInterval }} + scrapeInterval: {{ .Values.prometheus.prometheusSpec.scrapeInterval }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.prometheusSpec.scrapeTimeout }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.evaluationInterval }} + evaluationInterval: {{ .Values.prometheus.prometheusSpec.evaluationInterval }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.resources }} + resources: +{{ toYaml .Values.prometheus.prometheusSpec.resources | indent 4 }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} + retention: {{ .Values.prometheus.prometheusSpec.retention | quote }} +{{- if .Values.prometheus.prometheusSpec.retentionSize }} + retentionSize: {{ .Values.prometheus.prometheusSpec.retentionSize | quote }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.tsdb }} + tsdb: + {{- if .Values.prometheus.prometheusSpec.tsdb.outOfOrderTimeWindow }} + outOfOrderTimeWindow: {{ .Values.prometheus.prometheusSpec.tsdb.outOfOrderTimeWindow }} + {{- end }} +{{- end }} +{{- end }} +{{- if eq .Values.prometheus.prometheusSpec.walCompression false }} + walCompression: false +{{ else }} + walCompression: true +{{- end }} +{{- if .Values.prometheus.prometheusSpec.routePrefix }} + routePrefix: {{ .Values.prometheus.prometheusSpec.routePrefix | quote }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.secrets }} + secrets: +{{ toYaml .Values.prometheus.prometheusSpec.secrets | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.configMaps }} + configMaps: +{{ toYaml .Values.prometheus.prometheusSpec.configMaps | indent 4 }} +{{- end }} + serviceAccountName: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} +{{- if .Values.prometheus.prometheusSpec.serviceMonitorSelector }} + serviceMonitorSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.serviceMonitorSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues }} + serviceMonitorSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + serviceMonitorSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.serviceMonitorNamespaceSelector }} + serviceMonitorNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.serviceMonitorNamespaceSelector | indent 4) . }} +{{ else }} + serviceMonitorNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMonitorSelector }} + podMonitorSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.podMonitorSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues }} + podMonitorSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + podMonitorSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMonitorNamespaceSelector }} + podMonitorNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.podMonitorNamespaceSelector | indent 4) . }} +{{ else }} + podMonitorNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.probeSelector }} + probeSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.probeSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.probeSelectorNilUsesHelmValues }} + probeSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + probeSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.probeNamespaceSelector }} + probeNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.probeNamespaceSelector | indent 4) . }} +{{ else }} + probeNamespaceSelector: {} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) (or .Values.prometheus.prometheusSpec.remoteRead .Values.prometheus.prometheusSpec.additionalRemoteRead) }} + remoteRead: +{{- if .Values.prometheus.prometheusSpec.remoteRead }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.remoteRead | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalRemoteRead }} +{{ toYaml .Values.prometheus.prometheusSpec.additionalRemoteRead | indent 4 }} +{{- end }} +{{- end }} +{{- if (or .Values.prometheus.prometheusSpec.remoteWrite .Values.prometheus.prometheusSpec.additionalRemoteWrite) }} + remoteWrite: +{{- if .Values.prometheus.prometheusSpec.remoteWrite }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.remoteWrite | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalRemoteWrite }} +{{ toYaml .Values.prometheus.prometheusSpec.additionalRemoteWrite | indent 4 }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.securityContext }} + securityContext: +{{ toYaml .Values.prometheus.prometheusSpec.securityContext | indent 4 }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} +{{- if .Values.prometheus.prometheusSpec.ruleNamespaceSelector }} + ruleNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.ruleNamespaceSelector | indent 4) . }} +{{ else }} + ruleNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.ruleSelector }} + ruleSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.ruleSelector | indent 4) . }} +{{- else if .Values.prometheus.prometheusSpec.ruleSelectorNilUsesHelmValues }} + ruleSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + ruleSelector: {} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeConfigSelector }} + scrapeConfigSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.scrapeConfigSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.scrapeConfigSelectorNilUsesHelmValues }} + scrapeConfigSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + scrapeConfigSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeConfigNamespaceSelector }} + scrapeConfigNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.scrapeConfigNamespaceSelector | indent 4) . }} +{{ else }} + scrapeConfigNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.storageSpec }} + storage: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.storageSpec | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMetadata }} + podMetadata: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.podMetadata | indent 4) . }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.query }} + query: +{{ toYaml .Values.prometheus.prometheusSpec.query | indent 4}} +{{- end }} +{{- if or .Values.prometheus.prometheusSpec.podAntiAffinity .Values.prometheus.prometheusSpec.affinity }} + affinity: +{{- if .Values.prometheus.prometheusSpec.affinity }} +{{ toYaml .Values.prometheus.prometheusSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.prometheus.prometheusSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.prometheus.prometheusSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.prometheus.crname" . }}]} +{{- else if eq .Values.prometheus.prometheusSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.prometheus.prometheusSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.prometheus.crname" . }}]} +{{- end }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.prometheus.prometheusSpec.tolerations }} +{{ toYaml .Values.prometheus.prometheusSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.prometheus.prometheusSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalScrapeConfigs }} + additionalScrapeConfigs: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-scrape-confg + key: additional-scrape-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.enabled }} + additionalScrapeConfigs: + name: {{ .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.key }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} +{{- if or .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret }} + additionalAlertManagerConfigs: +{{- if .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs }} + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-confg + key: additional-alertmanager-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret }} + name: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.key }} + {{- if hasKey .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret "optional" }} + optional: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.optional }} + {{- end }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs }} + additionalAlertRelabelConfigs: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-relabel-confg + key: additional-alert-relabel-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigsSecret }} + additionalAlertRelabelConfigs: + name: {{ .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigsSecret.key }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.containers }} + containers: +{{ tpl .Values.prometheus.prometheusSpec.containers $ | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.initContainers }} + initContainers: +{{ toYaml .Values.prometheus.prometheusSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.priorityClassName }} + priorityClassName: {{ .Values.prometheus.prometheusSpec.priorityClassName }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} +{{- if .Values.prometheus.prometheusSpec.thanos }} + thanos: +{{- with (omit .Values.prometheus.prometheusSpec.thanos "objectStorageConfig")}} +{{ toYaml . | indent 4 }} +{{- end }} +{{- if ((.Values.prometheus.prometheusSpec.thanos.objectStorageConfig).existingSecret) }} + objectStorageConfig: + key: "{{.Values.prometheus.prometheusSpec.thanos.objectStorageConfig.existingSecret.key }}" + name: "{{.Values.prometheus.prometheusSpec.thanos.objectStorageConfig.existingSecret.name }}" +{{- else if ((.Values.prometheus.prometheusSpec.thanos.objectStorageConfig).secret) }} + objectStorageConfig: + key: object-storage-configs.yaml + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.disableCompaction }} + disableCompaction: {{ .Values.prometheus.prometheusSpec.disableCompaction }} +{{- end }} +{{- end }} + portName: {{ .Values.prometheus.prometheusSpec.portName }} +{{- if .Values.prometheus.prometheusSpec.volumes }} + volumes: +{{ toYaml .Values.prometheus.prometheusSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.prometheus.prometheusSpec.volumeMounts | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.arbitraryFSAccessThroughSMs }} + arbitraryFSAccessThroughSMs: +{{ toYaml .Values.prometheus.prometheusSpec.arbitraryFSAccessThroughSMs | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.overrideHonorLabels }} + overrideHonorLabels: {{ .Values.prometheus.prometheusSpec.overrideHonorLabels }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.overrideHonorTimestamps }} + overrideHonorTimestamps: {{ .Values.prometheus.prometheusSpec.overrideHonorTimestamps }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + ignoreNamespaceSelectors: {{ .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedNamespaceLabel }} + enforcedNamespaceLabel: {{ .Values.prometheus.prometheusSpec.enforcedNamespaceLabel }} +{{- $prometheusDefaultRulesExcludedFromEnforce := (include "rules.names" .) | fromYaml }} +{{- if not .Values.prometheus.agentMode }} + prometheusRulesExcludedFromEnforce: +{{- range $prometheusDefaultRulesExcludedFromEnforce.rules }} + - ruleNamespace: "{{ template "kube-prometheus-stack.namespace" $ }}" + ruleName: "{{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) . | trunc 63 | trimSuffix "-" }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.prometheusRulesExcludedFromEnforce }} +{{ toYaml .Values.prometheus.prometheusSpec.prometheusRulesExcludedFromEnforce | indent 4 }} +{{- end }} +{{- end }} + excludedFromEnforcement: +{{- range $prometheusDefaultRulesExcludedFromEnforce.rules }} + - group: monitoring.coreos.com + resource: prometheusrules + namespace: "{{ template "kube-prometheus-stack.namespace" $ }}" + name: "{{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) . | trunc 63 | trimSuffix "-" }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.excludedFromEnforcement }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.excludedFromEnforcement | indent 4) . }} +{{- end }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.queryLogFile }} + queryLogFile: {{ .Values.prometheus.prometheusSpec.queryLogFile }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.sampleLimit }} + sampleLimit: {{ .Values.prometheus.prometheusSpec.sampleLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedKeepDroppedTargets }} + enforcedKeepDroppedTargets: {{ .Values.prometheus.prometheusSpec.enforcedKeepDroppedTargets }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedSampleLimit }} + enforcedSampleLimit: {{ .Values.prometheus.prometheusSpec.enforcedSampleLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedTargetLimit }} + enforcedTargetLimit: {{ .Values.prometheus.prometheusSpec.enforcedTargetLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelLimit }} + enforcedLabelLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelNameLengthLimit }} + enforcedLabelNameLengthLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelNameLengthLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelValueLengthLimit}} + enforcedLabelValueLengthLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelValueLengthLimit }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.allowOverlappingBlocks }} + allowOverlappingBlocks: {{ .Values.prometheus.prometheusSpec.allowOverlappingBlocks }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.minReadySeconds }} + minReadySeconds: {{ .Values.prometheus.prometheusSpec.minReadySeconds }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.maximumStartupDurationSeconds }} + maximumStartupDurationSeconds: {{ .Values.prometheus.prometheusSpec.maximumStartupDurationSeconds }} +{{- end }} + hostNetwork: {{ .Values.prometheus.prometheusSpec.hostNetwork }} +{{- if .Values.prometheus.prometheusSpec.hostAliases }} + hostAliases: +{{ toYaml .Values.prometheus.prometheusSpec.hostAliases | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.tracingConfig }} + tracingConfig: +{{ toYaml .Values.prometheus.prometheusSpec.tracingConfig | indent 4 }} +{{- end }} +{{- with .Values.prometheus.prometheusSpec.additionalConfig }} + {{- tpl (toYaml .) $ | nindent 2 }} +{{- end }} +{{- with .Values.prometheus.prometheusSpec.additionalConfigString }} + {{- tpl . $ | nindent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/psp-clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/psp-clusterrole.yaml new file mode 100644 index 0000000000..71476cd18b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/psp-clusterrole.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.prometheus.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-prometheus +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/psp-clusterrolebinding.yaml new file mode 100644 index 0000000000..a393928c78 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/psp-clusterrolebinding.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.prometheus.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/psp.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/psp.yaml new file mode 100644 index 0000000000..62d3854151 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/psp.yaml @@ -0,0 +1,58 @@ +{{- if and .Values.prometheus.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' +{{- if .Values.prometheus.podSecurityPolicy.volumes }} +{{ toYaml .Values.prometheus.podSecurityPolicy.volumes | indent 4 }} +{{- end }} + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- if .Values.prometheus.podSecurityPolicy.allowedCapabilities }} + allowedCapabilities: +{{ toYaml .Values.prometheus.podSecurityPolicy.allowedCapabilities | indent 4 }} +{{- end }} +{{- if .Values.prometheus.podSecurityPolicy.allowedHostPaths }} + allowedHostPaths: +{{ toYaml .Values.prometheus.podSecurityPolicy.allowedHostPaths | indent 4 }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml new file mode 100644 index 0000000000..b66f052ade --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml @@ -0,0 +1,305 @@ +{{- /* +Generated from 'alertmanager.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/alertmanager-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.alertmanager }} +{{- $alertmanagerJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: alertmanager.rules + rules: +{{- if not (.Values.defaultRules.disabled.AlertmanagerFailedReload | default false) }} + - alert: AlertmanagerFailedReload + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Configuration has failed to load for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerfailedreload + summary: Reloading an Alertmanager configuration has failed. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(alertmanager_config_last_reload_successful{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) == 0 + for: {{ dig "AlertmanagerFailedReload" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerFailedReload" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerMembersInconsistent | default false) }} + - alert: AlertmanagerMembersInconsistent + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Alertmanager {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} has only found {{`{{`}} $value {{`}}`}} members of the {{`{{`}}$labels.job{{`}}`}} cluster. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagermembersinconsistent + summary: A member of an Alertmanager cluster has not found all other cluster members. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + < on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) group_left + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) (max_over_time(alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m])) + for: {{ dig "AlertmanagerMembersInconsistent" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerMembersInconsistent" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerFailedToSendAlerts | default false) }} + - alert: AlertmanagerFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Alertmanager {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} failed to send {{`{{`}} $value | humanizePercentage {{`}}`}} of notifications to {{`{{`}} $labels.integration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerfailedtosendalerts + summary: An Alertmanager instance failed to send notifications. + expr: |- + ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + / + ignoring (reason) group_left rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + ) + > 0.01 + for: {{ dig "AlertmanagerFailedToSendAlerts" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerFailedToSendAlerts" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterFailedToSendAlerts | default false) }} + - alert: AlertmanagerClusterFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: The minimum notification failure rate to {{`{{`}} $labels.integration {{`}}`}} sent from any instance in the {{`{{`}}$labels.job{{`}}`}} cluster is {{`{{`}} $value | humanizePercentage {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterfailedtosendalerts + summary: All Alertmanager instances in a cluster failed to send notifications to a critical integration. + expr: |- + min by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service, integration) ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration=~`.*`}[5m]) + / + ignoring (reason) group_left rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration=~`.*`}[5m]) + ) + > 0.01 + for: {{ dig "AlertmanagerClusterFailedToSendAlerts" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterFailedToSendAlerts" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterFailedToSendAlerts | default false) }} + - alert: AlertmanagerClusterFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: The minimum notification failure rate to {{`{{`}} $labels.integration {{`}}`}} sent from any instance in the {{`{{`}}$labels.job{{`}}`}} cluster is {{`{{`}} $value | humanizePercentage {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterfailedtosendalerts + summary: All Alertmanager instances in a cluster failed to send notifications to a non-critical integration. + expr: |- + min by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service, integration) ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration!~`.*`}[5m]) + / + ignoring (reason) group_left rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration!~`.*`}[5m]) + ) + > 0.01 + for: {{ dig "AlertmanagerClusterFailedToSendAlerts" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterFailedToSendAlerts" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerConfigInconsistent | default false) }} + - alert: AlertmanagerConfigInconsistent + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have different configurations. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerconfiginconsistent + summary: Alertmanager instances within the same cluster have different configurations. + expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + count_values by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ("config_hash", alertmanager_config_hash{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}) + ) + != 1 + for: {{ dig "AlertmanagerConfigInconsistent" "for" "20m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerConfigInconsistent" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterDown | default false) }} + - alert: AlertmanagerClusterDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have been up for less than half of the last 5m.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterdown + summary: Half or more of the Alertmanager instances within the same cluster are down. + expr: |- + ( + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + avg_over_time(up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) < 0.5 + ) + / + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} + ) + ) + >= 0.5 + for: {{ dig "AlertmanagerClusterDown" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterCrashlooping | default false) }} + - alert: AlertmanagerClusterCrashlooping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have restarted at least 5 times in the last 10m.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclustercrashlooping + summary: Half or more of the Alertmanager instances within the same cluster are crashlooping. + expr: |- + ( + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + changes(process_start_time_seconds{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[10m]) > 4 + ) + / + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} + ) + ) + >= 0.5 + for: {{ dig "AlertmanagerClusterCrashlooping" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterCrashlooping" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/config-reloaders.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/config-reloaders.yaml new file mode 100644 index 0000000000..8416d6df40 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/config-reloaders.yaml @@ -0,0 +1,57 @@ +{{- /* +Generated from 'config-reloaders' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/prometheusOperator-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.configReloaders }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "config-reloaders" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: config-reloaders + rules: +{{- if not (.Values.defaultRules.disabled.ConfigReloaderSidecarErrors | default false) }} + - alert: ConfigReloaderSidecarErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.configReloaders }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.configReloaders | indent 8 }} +{{- end }} + description: 'Errors encountered while the {{`{{`}}$labels.pod{{`}}`}} config-reloader sidecar attempts to sync config in {{`{{`}}$labels.namespace{{`}}`}} namespace. + + As a result, configuration for service running in {{`{{`}}$labels.pod{{`}}`}} may be stale and cannot be updated anymore.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/configreloadersidecarerrors + summary: config-reloader sidecar has not had a successful reload for 10m + expr: max_over_time(reloader_last_reload_successful{namespace=~".+"}[5m]) == 0 + for: {{ dig "ConfigReloaderSidecarErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "ConfigReloaderSidecarErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.configReloaders }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.configReloaders }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/etcd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/etcd.yaml new file mode 100644 index 0000000000..a1d7a508f8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/etcd.yaml @@ -0,0 +1,461 @@ +{{- /* +Generated from 'etcd' group from https://github.com/etcd-io/etcd.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.etcd }} +{{- if (include "exporter.kubeEtcd.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "etcd" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: etcd + rules: +{{- if not (.Values.defaultRules.disabled.etcdMembersDown | default false) }} + - alert: etcdMembersDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": members are down ({{`{{`}} $value {{`}}`}}).' + summary: etcd cluster members are down. + expr: |- + max without (endpoint) ( + sum without (instance) (up{job=~".*etcd.*"} == bool 0) + or + count without (To) ( + sum without (instance) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s])) > 0.01 + ) + ) + > 0 + for: {{ dig "etcdMembersDown" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdMembersDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdInsufficientMembers | default false) }} + - alert: etcdInsufficientMembers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": insufficient members ({{`{{`}} $value {{`}}`}}).' + summary: etcd cluster has insufficient number of members. + expr: sum(up{job=~".*etcd.*"} == bool 1) without (instance) < ((count(up{job=~".*etcd.*"}) without (instance) + 1) / 2) + for: {{ dig "etcdInsufficientMembers" "for" "3m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdInsufficientMembers" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdNoLeader | default false) }} + - alert: etcdNoLeader + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member {{`{{`}} $labels.instance {{`}}`}} has no leader.' + summary: etcd cluster has no leader. + expr: etcd_server_has_leader{job=~".*etcd.*"} == 0 + for: {{ dig "etcdNoLeader" "for" "1m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdNoLeader" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfLeaderChanges | default false) }} + - alert: etcdHighNumberOfLeaderChanges + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated.' + summary: etcd cluster has high number of leader changes. + expr: increase((max without (instance) (etcd_server_leader_changes_seen_total{job=~".*etcd.*"}) or 0*absent(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}))[15m:1m]) >= 4 + for: {{ dig "etcdHighNumberOfLeaderChanges" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfLeaderChanges" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedGRPCRequests | default false) }} + - alert: etcdHighNumberOfFailedGRPCRequests + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster has high number of failed grpc requests. + expr: |- + 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code) + / + sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code) + > 1 + for: {{ dig "etcdHighNumberOfFailedGRPCRequests" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfFailedGRPCRequests" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedGRPCRequests | default false) }} + - alert: etcdHighNumberOfFailedGRPCRequests + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster has high number of failed grpc requests. + expr: |- + 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code) + / + sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code) + > 5 + for: {{ dig "etcdHighNumberOfFailedGRPCRequests" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfFailedGRPCRequests" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdGRPCRequestsSlow | default false) }} + - alert: etcdGRPCRequestsSlow + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile of gRPC requests is {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}} for {{`{{`}} $labels.grpc_method {{`}}`}} method.' + summary: etcd grpc requests are slow + expr: |- + histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_method!="Defragment", grpc_type="unary"}[5m])) without(grpc_type)) + > 0.15 + for: {{ dig "etcdGRPCRequestsSlow" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdGRPCRequestsSlow" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdMemberCommunicationSlow | default false) }} + - alert: etcdMemberCommunicationSlow + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member communication with {{`{{`}} $labels.To {{`}}`}} is taking {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster member communication is slow. + expr: |- + histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.15 + for: {{ dig "etcdMemberCommunicationSlow" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdMemberCommunicationSlow" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedProposals | default false) }} + - alert: etcdHighNumberOfFailedProposals + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} proposal failures within the last 30 minutes on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster has high number of proposal failures. + expr: rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5 + for: {{ dig "etcdHighNumberOfFailedProposals" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfFailedProposals" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighFsyncDurations | default false) }} + - alert: etcdHighFsyncDurations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fsync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster 99th percentile fsync durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.5 + for: {{ dig "etcdHighFsyncDurations" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighFsyncDurations" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighFsyncDurations | default false) }} + - alert: etcdHighFsyncDurations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fsync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster 99th percentile fsync durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 1 + for: {{ dig "etcdHighFsyncDurations" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighFsyncDurations" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighCommitDurations | default false) }} + - alert: etcdHighCommitDurations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile commit durations {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster 99th percentile commit durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.25 + for: {{ dig "etcdHighCommitDurations" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighCommitDurations" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdDatabaseQuotaLowSpace | default false) }} + - alert: etcdDatabaseQuotaLowSpace + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": database size exceeds the defined quota on etcd instance {{`{{`}} $labels.instance {{`}}`}}, please defrag or increase the quota as the writes to etcd will be disabled when it is full.' + summary: etcd cluster database is running full. + expr: (last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_server_quota_backend_bytes{job=~".*etcd.*"}[5m]))*100 > 95 + for: {{ dig "etcdDatabaseQuotaLowSpace" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdDatabaseQuotaLowSpace" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdExcessiveDatabaseGrowth | default false) }} + - alert: etcdExcessiveDatabaseGrowth + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": Predicting running out of disk space in the next four hours, based on write observations within the past four hours on etcd instance {{`{{`}} $labels.instance {{`}}`}}, please check as it might be disruptive.' + summary: etcd cluster database growing very fast. + expr: predict_linear(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[4h], 4*60*60) > etcd_server_quota_backend_bytes{job=~".*etcd.*"} + for: {{ dig "etcdExcessiveDatabaseGrowth" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdExcessiveDatabaseGrowth" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdDatabaseHighFragmentationRatio | default false) }} + - alert: etcdDatabaseHighFragmentationRatio + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": database size in use on instance {{`{{`}} $labels.instance {{`}}`}} is {{`{{`}} $value | humanizePercentage {{`}}`}} of the actual allocated disk space, please run defragmentation (e.g. etcdctl defrag) to retrieve the unused fragmented disk space.' + runbook_url: https://etcd.io/docs/v3.5/op-guide/maintenance/#defragmentation + summary: etcd database size in use is less than 50% of the actual allocated storage. + expr: (last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m])) < 0.5 and etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"} > 104857600 + for: {{ dig "etcdDatabaseHighFragmentationRatio" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdDatabaseHighFragmentationRatio" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/general.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/general.rules.yaml new file mode 100644 index 0000000000..8aca0b85f5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/general.rules.yaml @@ -0,0 +1,125 @@ +{{- /* +Generated from 'general.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.general }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "general.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: general.rules + rules: +{{- if not (.Values.defaultRules.disabled.TargetDown | default false) }} + - alert: TargetDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.general }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.general | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.4g" $value {{`}}`}}% of the {{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.service {{`}}`}} targets in {{`{{`}} $labels.namespace {{`}}`}} namespace are down.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/targetdown + summary: One or more targets are unreachable. + expr: 100 * (count(up == 0) BY (cluster, job, namespace, service) / count(up) BY (cluster, job, namespace, service)) > 10 + for: {{ dig "TargetDown" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "TargetDown" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.Watchdog | default false) }} + - alert: Watchdog + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.general }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.general | indent 8 }} +{{- end }} + description: 'This is an alert meant to ensure that the entire alerting pipeline is functional. + + This alert is always firing, therefore it should always be firing in Alertmanager + + and always fire against a receiver. There are integrations with various notification + + mechanisms that send a notification when this alert is not firing. For example the + + "DeadMansSnitch" integration in PagerDuty. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/watchdog + summary: An alert that should always be firing to certify that Alertmanager is working properly. + expr: vector(1) + labels: + severity: {{ dig "Watchdog" "severity" "none" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.InfoInhibitor | default false) }} + - alert: InfoInhibitor + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.general }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.general | indent 8 }} +{{- end }} + description: 'This is an alert that is used to inhibit info alerts. + + By themselves, the info-level alerts are sometimes very noisy, but they are relevant when combined with + + other alerts. + + This alert fires whenever there''s a severity="info" alert, and stops firing when another alert with a + + severity of ''warning'' or ''critical'' starts firing on the same namespace. + + This alert should be routed to a null receiver and configured to inhibit alerts with severity="info". + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/infoinhibitor + summary: Info-level alert inhibition. + expr: ALERTS{severity = "info"} == 1 unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace) ALERTS{alertname != "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} == 1 + labels: + severity: {{ dig "InfoInhibitor" "severity" "none" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml new file mode 100644 index 0000000000..9de5f5bc9c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml @@ -0,0 +1,43 @@ +{{- /* +Generated from 'k8s.rules.container-cpu-usage-seconds-total' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerCpuUsageSecondsTotal }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-cpu-usage-seconds-total" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_cpu_usage_seconds_total + rules: + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + irate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}[5m]) + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) ( + 1, max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerCpuUsageSecondsTotal }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerCpuUsageSecondsTotal }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml new file mode 100644 index 0000000000..323f41f9cb --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-cache' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemoryCache }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-cache" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_cache + rules: + - expr: |- + container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_cache + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryCache }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryCache }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml new file mode 100644 index 0000000000..312d73c889 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-rss' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemoryRss }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-rss" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_rss + rules: + - expr: |- + container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_rss + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryRss }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryRss }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml new file mode 100644 index 0000000000..136595e801 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-swap' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemorySwap }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-swap" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_swap + rules: + - expr: |- + container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_swap + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemorySwap }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemorySwap }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml new file mode 100644 index 0000000000..d308b7473a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-working-set-bytes' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemoryWorkingSetBytes }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-working-set-bytes" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_working_set_bytes + rules: + - expr: |- + container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_working_set_bytes + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryWorkingSetBytes }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryWorkingSetBytes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml new file mode 100644 index 0000000000..2d896e59e4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml @@ -0,0 +1,168 @@ +{{- /* +Generated from 'k8s.rules.container-resource' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerResource }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-resource" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_resource + rules: + - expr: |- + kube_pod_container_resource_requests{resource="memory",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_requests + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="memory",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_requests:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + kube_pod_container_resource_requests{resource="cpu",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="cpu",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_requests:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="memory",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_limits + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="memory",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_limits:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="cpu",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="cpu",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_limits:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml new file mode 100644 index 0000000000..4915b25e73 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml @@ -0,0 +1,107 @@ +{{- /* +Generated from 'k8s.rules.pod-owner' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sPodOwner }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.pod-owner" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.pod_owner + rules: + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="ReplicaSet"}, + "replicaset", "$1", "owner_name", "(.*)" + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}replicaset, namespace) group_left(owner_name) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}replicaset, namespace) ( + 1, max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}replicaset, namespace, owner_name) ( + kube_replicaset_owner{job="{{ $kubeStateMetricsJob }}"} + ) + ), + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: deployment + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="DaemonSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: daemonset + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="StatefulSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: statefulset + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="Job"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: job + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.yaml new file mode 100644 index 0000000000..c61bd222ab --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.yaml @@ -0,0 +1,237 @@ +{{- /* +Generated from 'k8s.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8s }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules + rules: + - expr: |- + sum by (cluster, namespace, pod, container) ( + irate(container_cpu_usage_seconds_total{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics/cadvisor", image!=""}[5m]) + ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( + 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_working_set_bytes + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_rss + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_cache + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_swap + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_requests + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_requests:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_requests:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_limits + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_limits:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_limits:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="ReplicaSet"}, + "replicaset", "$1", "owner_name", "(.*)" + ) * on(replicaset, namespace) group_left(owner_name) topk by(replicaset, namespace) ( + 1, max by (replicaset, namespace, owner_name) ( + kube_replicaset_owner{job="kube-state-metrics"} + ) + ), + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: deployment + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="DaemonSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: daemonset + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="StatefulSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: statefulset + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="Job"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: job + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml new file mode 100644 index 0000000000..6194e9c614 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml @@ -0,0 +1,273 @@ +{{- /* +Generated from 'kube-apiserver-availability.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverAvailability }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-availability.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - interval: 3m + name: kube-apiserver-availability.rules + rules: + - expr: avg_over_time(code_verb:apiserver_request_total:increase1h[30d]) * 24 * 30 + record: code_verb:apiserver_request_total:increase30d + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"LIST|GET"}) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code:apiserver_request_total:increase30d + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code:apiserver_request_total:increase30d + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope) (increase(apiserver_request_sli_duration_seconds_count{job="apiserver"}[1h])) + record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope) (avg_over_time(cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase1h[30d]) * 24 * 30) + record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope, le) (increase(apiserver_request_sli_duration_seconds_bucket[1h])) + record: cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope, le) (avg_over_time(cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h[30d]) * 24 * 30) + record: cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - ( + ( + # write too slow + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + - + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"}) + ) + + ( + # read too slow + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"LIST|GET"}) + - + ( + ( + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"}) + or + vector(0) + ) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"}) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"}) + ) + ) + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{code=~"5.."} or vector(0)) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d) + labels: + verb: all + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:availability30d + - expr: |- + 1 - ( + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"LIST|GET"}) + - + ( + # too slow + ( + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"}) + or + vector(0) + ) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"}) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"}) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="read",code=~"5.."} or vector(0)) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="read"}) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:availability30d + - expr: |- + 1 - ( + ( + # too slow + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + - + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"}) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="write",code=~"5.."} or vector(0)) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="write"}) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:availability30d + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code_resource:apiserver_request_total:rate5m + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code_resource:apiserver_request_total:rate5m + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"2.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"3.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"4.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"5.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml new file mode 100644 index 0000000000..e6666a6f41 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml @@ -0,0 +1,440 @@ +{{- /* +Generated from 'kube-apiserver-burnrate.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverBurnrate }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-burnrate.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver-burnrate.rules + rules: + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1d])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1d])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1d])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1d])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1d])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1h])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1h])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1h])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[2h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[2h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[2h])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[2h])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[2h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[2h])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate2h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[30m])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[30m])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[30m])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[30m])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[30m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[30m])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate30m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[3d])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[3d])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[3d])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[3d])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[3d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[3d])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate3d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[5m])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[5m])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[5m])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[5m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate5m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[6h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[6h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[6h])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[6h])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[6h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[6h])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate6h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1d])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1d])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1d])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1h])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1h])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[2h])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[2h])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[2h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate2h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[30m])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[30m])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[30m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate30m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[3d])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[3d])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[3d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3d])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate3d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[5m])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[5m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate5m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[6h])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[6h])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[6h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate6h +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml new file mode 100644 index 0000000000..d145341952 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml @@ -0,0 +1,53 @@ +{{- /* +Generated from 'kube-apiserver-histogram.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverHistogram }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-histogram.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver-histogram.rules + rules: + - expr: histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0 + labels: + quantile: '0.99' + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, le, resource) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0 + labels: + quantile: '0.99' + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml new file mode 100644 index 0000000000..30ef9a4293 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml @@ -0,0 +1,159 @@ +{{- /* +Generated from 'kube-apiserver-slos' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverSlos }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-slos" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver-slos + rules: +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate1h) > (14.40 * 0.01000) + and + sum(apiserver_request:burnrate5m) > (14.40 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "2m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 1h + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "critical" .Values.customRules }} + short: 5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate6h) > (6.00 * 0.01000) + and + sum(apiserver_request:burnrate30m) > (6.00 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 6h + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "critical" .Values.customRules }} + short: 30m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate1d) > (3.00 * 0.01000) + and + sum(apiserver_request:burnrate2h) > (3.00 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 1d + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "warning" .Values.customRules }} + short: 2h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate3d) > (1.00 * 0.01000) + and + sum(apiserver_request:burnrate6h) > (1.00 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "3h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 3d + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "warning" .Values.customRules }} + short: 6h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml new file mode 100644 index 0000000000..fcf35f389b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml @@ -0,0 +1,49 @@ +{{- /* +Generated from 'kube-prometheus-general.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubePrometheusGeneral }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-prometheus-general.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-prometheus-general.rules + rules: + - expr: count without(instance, pod, node) (up == 1) + record: count:up1 + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: count without(instance, pod, node) (up == 0) + record: count:up0 + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml new file mode 100644 index 0000000000..7a0d202324 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml @@ -0,0 +1,93 @@ +{{- /* +Generated from 'kube-prometheus-node-recording.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubePrometheusNodeRecording }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-prometheus-node-recording.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-prometheus-node-recording.rules + rules: + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[3m])) BY (instance) + record: instance:node_cpu:rate:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_network_receive_bytes_total[3m])) BY (instance) + record: instance:node_network_receive_bytes:rate:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_network_transmit_bytes_total[3m])) BY (instance) + record: instance:node_network_transmit_bytes:rate:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) WITHOUT (cpu, mode) / ON(instance) GROUP_LEFT() count(sum(node_cpu_seconds_total) BY (instance, cpu)) BY (instance) + record: instance:node_cpu:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) + record: cluster:node_cpu:sum_rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: cluster:node_cpu:sum_rate5m / count(sum(node_cpu_seconds_total) BY (instance, cpu)) + record: cluster:node_cpu:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml new file mode 100644 index 0000000000..c9d61ce37b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml @@ -0,0 +1,135 @@ +{{- /* +Generated from 'kube-scheduler.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeScheduler.enabled .Values.defaultRules.rules.kubeSchedulerRecording }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-scheduler.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-scheduler.rules + rules: + - expr: histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_binding_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_binding_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-state-metrics.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-state-metrics.yaml new file mode 100644 index 0000000000..d1ad3cae5e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kube-state-metrics.yaml @@ -0,0 +1,152 @@ +{{- /* +Generated from 'kube-state-metrics' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubeStateMetrics-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeStateMetrics }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-state-metrics" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-state-metrics + rules: +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsListErrors | default false) }} + - alert: KubeStateMetricsListErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics is experiencing errors at an elevated rate in list operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricslisterrors + summary: kube-state-metrics is experiencing errors in list operations. + expr: |- + (sum(rate(kube_state_metrics_list_total{job="{{ $kubeStateMetricsJob }}",result="error"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(rate(kube_state_metrics_list_total{job="{{ $kubeStateMetricsJob }}"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) + > 0.01 + for: {{ dig "KubeStateMetricsListErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsListErrors" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsWatchErrors | default false) }} + - alert: KubeStateMetricsWatchErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics is experiencing errors at an elevated rate in watch operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricswatcherrors + summary: kube-state-metrics is experiencing errors in watch operations. + expr: |- + (sum(rate(kube_state_metrics_watch_total{job="{{ $kubeStateMetricsJob }}",result="error"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(rate(kube_state_metrics_watch_total{job="{{ $kubeStateMetricsJob }}"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) + > 0.01 + for: {{ dig "KubeStateMetricsWatchErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsWatchErrors" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsShardingMismatch | default false) }} + - alert: KubeStateMetricsShardingMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics pods are running with different --total-shards configuration, some Kubernetes objects may be exposed multiple times or not exposed at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricsshardingmismatch + summary: kube-state-metrics sharding is misconfigured. + expr: stdvar (kube_state_metrics_total_shards{job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) != 0 + for: {{ dig "KubeStateMetricsShardingMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsShardingMismatch" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsShardsMissing | default false) }} + - alert: KubeStateMetricsShardsMissing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics shards are missing, some Kubernetes objects are not being exposed. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricsshardsmissing + summary: kube-state-metrics shards are missing. + expr: |- + 2^max(kube_state_metrics_total_shards{job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - 1 + - + sum( 2 ^ max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, shard_ordinal) (kube_state_metrics_shard_ordinal{job="{{ $kubeStateMetricsJob }}"}) ) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + != 0 + for: {{ dig "KubeStateMetricsShardsMissing" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsShardsMissing" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubelet.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubelet.rules.yaml new file mode 100644 index 0000000000..39fdddf3fe --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubelet.rules.yaml @@ -0,0 +1,65 @@ +{{- /* +Generated from 'kubelet.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubelet }} +{{- if (include "exporter.kubelet.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubelet.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubelet.rules + rules: + - expr: histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, le) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, le) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, le) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml new file mode 100644 index 0000000000..2a861a522c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml @@ -0,0 +1,568 @@ +{{- /* +Generated from 'kubernetes-apps' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesApps }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-apps" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-apps + rules: +{{- if not (.Values.defaultRules.disabled.KubePodCrashLooping | default false) }} + - alert: KubePodCrashLooping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: 'Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} ({{`{{`}} $labels.container {{`}}`}}) is in waiting state (reason: "CrashLoopBackOff").' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepodcrashlooping + summary: Pod is crash looping. + expr: max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff", job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[5m]) >= 1 + for: {{ dig "KubePodCrashLooping" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePodCrashLooping" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePodNotReady | default false) }} + - alert: KubePodNotReady + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} has been in a non-ready state for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepodnotready + summary: Pod has been in a non-ready state for more than 15 minutes. + expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}", phase=~"Pending|Unknown|Failed"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left(owner_kind) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + 1, max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"}) + ) + ) > 0 + for: {{ dig "KubePodNotReady" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePodNotReady" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDeploymentGenerationMismatch | default false) }} + - alert: KubeDeploymentGenerationMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Deployment generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} does not match, this indicates that the Deployment has failed but has not been rolled back. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedeploymentgenerationmismatch + summary: Deployment generation mismatch due to possible roll-back + expr: |- + kube_deployment_status_observed_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_deployment_metadata_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + for: {{ dig "KubeDeploymentGenerationMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDeploymentGenerationMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDeploymentReplicasMismatch | default false) }} + - alert: KubeDeploymentReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Deployment {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedeploymentreplicasmismatch + summary: Deployment has not matched the expected number of replicas. + expr: |- + ( + kube_deployment_spec_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + > + kube_deployment_status_replicas_available{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) and ( + changes(kube_deployment_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[10m]) + == + 0 + ) + for: {{ dig "KubeDeploymentReplicasMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDeploymentReplicasMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDeploymentRolloutStuck | default false) }} + - alert: KubeDeploymentRolloutStuck + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Rollout of deployment {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} is not progressing for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedeploymentrolloutstuck + summary: Deployment rollout is not progressing. + expr: |- + kube_deployment_status_condition{condition="Progressing", status="false",job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != 0 + for: {{ dig "KubeDeploymentRolloutStuck" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDeploymentRolloutStuck" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetReplicasMismatch | default false) }} + - alert: KubeStatefulSetReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetreplicasmismatch + summary: StatefulSet has not matched the expected number of replicas. + expr: |- + ( + kube_statefulset_status_replicas_ready{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_status_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) and ( + changes(kube_statefulset_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[10m]) + == + 0 + ) + for: {{ dig "KubeStatefulSetReplicasMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStatefulSetReplicasMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetGenerationMismatch | default false) }} + - alert: KubeStatefulSetGenerationMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: StatefulSet generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} does not match, this indicates that the StatefulSet has failed but has not been rolled back. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetgenerationmismatch + summary: StatefulSet generation mismatch due to possible roll-back + expr: |- + kube_statefulset_status_observed_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_metadata_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + for: {{ dig "KubeStatefulSetGenerationMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStatefulSetGenerationMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetUpdateNotRolledOut | default false) }} + - alert: KubeStatefulSetUpdateNotRolledOut + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} update has not been rolled out. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetupdatenotrolledout + summary: StatefulSet update has not been rolled out. + expr: |- + ( + max without (revision) ( + kube_statefulset_status_current_revision{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + unless + kube_statefulset_status_update_revision{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) + * + ( + kube_statefulset_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) + ) and ( + changes(kube_statefulset_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[5m]) + == + 0 + ) + for: {{ dig "KubeStatefulSetUpdateNotRolledOut" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStatefulSetUpdateNotRolledOut" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetRolloutStuck | default false) }} + - alert: KubeDaemonSetRolloutStuck + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} has not finished or progressed for at least 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetrolloutstuck + summary: DaemonSet rollout is stuck. + expr: |- + ( + ( + kube_daemonset_status_current_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) or ( + kube_daemonset_status_number_misscheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + 0 + ) or ( + kube_daemonset_status_updated_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) or ( + kube_daemonset_status_number_available{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) + ) and ( + changes(kube_daemonset_status_updated_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[5m]) + == + 0 + ) + for: {{ dig "KubeDaemonSetRolloutStuck" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDaemonSetRolloutStuck" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeContainerWaiting | default false) }} + - alert: KubeContainerWaiting + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: pod/{{`{{`}} $labels.pod {{`}}`}} in namespace {{`{{`}} $labels.namespace {{`}}`}} on container {{`{{`}} $labels.container{{`}}`}} has been in waiting state for longer than 1 hour. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecontainerwaiting + summary: Pod container waiting longer than 1 hour + expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) (kube_pod_container_status_waiting_reason{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) > 0 + for: {{ dig "KubeContainerWaiting" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeContainerWaiting" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetNotScheduled | default false) }} + - alert: KubeDaemonSetNotScheduled + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are not scheduled.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetnotscheduled + summary: DaemonSet pods are not scheduled. + expr: |- + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + - + kube_daemonset_status_current_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0 + for: {{ dig "KubeDaemonSetNotScheduled" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDaemonSetNotScheduled" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetMisScheduled | default false) }} + - alert: KubeDaemonSetMisScheduled + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are running where they are not supposed to run.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetmisscheduled + summary: DaemonSet pods are misscheduled. + expr: kube_daemonset_status_number_misscheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0 + for: {{ dig "KubeDaemonSetMisScheduled" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDaemonSetMisScheduled" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeJobNotCompleted | default false) }} + - alert: KubeJobNotCompleted + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} is taking more than {{`{{`}} "43200" | humanizeDuration {{`}}`}} to complete. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubejobnotcompleted + summary: Job did not complete in time + expr: |- + time() - max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, job_name, cluster) (kube_job_status_start_time{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + and + kube_job_status_active{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0) > 43200 + labels: + severity: {{ dig "KubeJobNotCompleted" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeJobFailed | default false) }} + - alert: KubeJobFailed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} failed to complete. Removing failed job after investigation should clear this alert. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubejobfailed + summary: Job failed to complete. + expr: kube_job_failed{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0 + for: {{ dig "KubeJobFailed" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeJobFailed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeHpaReplicasMismatch | default false) }} + - alert: KubeHpaReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: HPA {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.horizontalpodautoscaler {{`}}`}} has not matched the desired number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubehpareplicasmismatch + summary: HPA has not matched desired number of replicas. + expr: |- + (kube_horizontalpodautoscaler_status_desired_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) + and + (kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + > + kube_horizontalpodautoscaler_spec_min_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) + and + (kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + < + kube_horizontalpodautoscaler_spec_max_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) + and + changes(kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[15m]) == 0 + for: {{ dig "KubeHpaReplicasMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeHpaReplicasMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeHpaMaxedOut | default false) }} + - alert: KubeHpaMaxedOut + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: HPA {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.horizontalpodautoscaler {{`}}`}} has been running at max replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubehpamaxedout + summary: HPA is running at max replicas + expr: |- + kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + == + kube_horizontalpodautoscaler_spec_max_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + for: {{ dig "KubeHpaMaxedOut" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeHpaMaxedOut" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-resources.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-resources.yaml new file mode 100644 index 0000000000..1d32f9bbad --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-resources.yaml @@ -0,0 +1,282 @@ +{{- /* +Generated from 'kubernetes-resources' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesResources }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-resources" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-resources + rules: +{{- if not (.Values.defaultRules.disabled.KubeCPUOvercommit | default false) }} + - alert: KubeCPUOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted CPU resource requests for Pods by {{`{{`}} $value {{`}}`}} CPU shares and cannot tolerate node failure. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecpuovercommit + summary: Cluster has overcommitted CPU resource requests. + expr: |- + sum(namespace_cpu:kube_pod_container_resource_requests:sum{job="{{ $kubeStateMetricsJob }}",}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - (sum(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + and + (sum(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + for: {{ dig "KubeCPUOvercommit" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeCPUOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeMemoryOvercommit | default false) }} + - alert: KubeMemoryOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted memory resource requests for Pods by {{`{{`}} $value | humanize {{`}}`}} bytes and cannot tolerate node failure. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubememoryovercommit + summary: Cluster has overcommitted memory resource requests. + expr: |- + sum(namespace_memory:kube_pod_container_resource_requests:sum{}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - (sum(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + and + (sum(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + for: {{ dig "KubeMemoryOvercommit" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeMemoryOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeCPUQuotaOvercommit | default false) }} + - alert: KubeCPUQuotaOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted CPU resource requests for Namespaces. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecpuquotaovercommit + summary: Cluster has overcommitted CPU resource requests. + expr: |- + sum(min without(resource) (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard", resource=~"(cpu|requests.cpu)"})) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(kube_node_status_allocatable{resource="cpu", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + > 1.5 + for: {{ dig "KubeCPUQuotaOvercommit" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeCPUQuotaOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeMemoryQuotaOvercommit | default false) }} + - alert: KubeMemoryQuotaOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted memory resource requests for Namespaces. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubememoryquotaovercommit + summary: Cluster has overcommitted memory resource requests. + expr: |- + sum(min without(resource) (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard", resource=~"(memory|requests.memory)"})) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + > 1.5 + for: {{ dig "KubeMemoryQuotaOvercommit" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeMemoryQuotaOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeQuotaAlmostFull | default false) }} + - alert: KubeQuotaAlmostFull + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubequotaalmostfull + summary: Namespace quota is going to be full. + expr: |- + kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard"} > 0) + > 0.9 < 1 + for: {{ dig "KubeQuotaAlmostFull" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeQuotaAlmostFull" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeQuotaFullyUsed | default false) }} + - alert: KubeQuotaFullyUsed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubequotafullyused + summary: Namespace quota is fully used. + expr: |- + kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard"} > 0) + == 1 + for: {{ dig "KubeQuotaFullyUsed" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeQuotaFullyUsed" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeQuotaExceeded | default false) }} + - alert: KubeQuotaExceeded + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubequotaexceeded + summary: Namespace quota has exceeded the limits. + expr: |- + kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard"} > 0) + > 1 + for: {{ dig "KubeQuotaExceeded" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeQuotaExceeded" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.CPUThrottlingHigh | default false) }} + - alert: CPUThrottlingHigh + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} throttling of CPU in namespace {{`{{`}} $labels.namespace {{`}}`}} for container {{`{{`}} $labels.container {{`}}`}} in pod {{`{{`}} $labels.pod {{`}}`}}.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/cputhrottlinghigh + summary: Processes experience elevated CPU throttling. + expr: |- + sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, container, pod, namespace) + / + sum(increase(container_cpu_cfs_periods_total{}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, container, pod, namespace) + > ( 25 / 100 ) + for: {{ dig "CPUThrottlingHigh" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "CPUThrottlingHigh" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml new file mode 100644 index 0000000000..b988445653 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml @@ -0,0 +1,216 @@ +{{- /* +Generated from 'kubernetes-storage' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesStorage }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-storage" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-storage + rules: +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeFillingUp | default false) }} + - alert: KubePersistentVolumeFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} is only {{`{{`}} $value | humanizePercentage {{`}}`}} free. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumefillingup + summary: PersistentVolume is filling up. + expr: |- + kubelet_volume_stats_available_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + < 0.03 + and + kubelet_volume_stats_used_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeFillingUp" "for" "1m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeFillingUp | default false) }} + - alert: KubePersistentVolumeFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} is expected to fill up within four days. Currently {{`{{`}} $value | humanizePercentage {{`}}`}} is available. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumefillingup + summary: PersistentVolume is filling up. + expr: |- + ( + kubelet_volume_stats_available_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.15 + and + kubelet_volume_stats_used_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + and + predict_linear(kubelet_volume_stats_available_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeInodesFillingUp | default false) }} + - alert: KubePersistentVolumeInodesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} only has {{`{{`}} $value | humanizePercentage {{`}}`}} free inodes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeinodesfillingup + summary: PersistentVolumeInodes are filling up. + expr: |- + ( + kubelet_volume_stats_inodes_free{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_inodes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.03 + and + kubelet_volume_stats_inodes_used{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeInodesFillingUp" "for" "1m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeInodesFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeInodesFillingUp | default false) }} + - alert: KubePersistentVolumeInodesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} is expected to run out of inodes within four days. Currently {{`{{`}} $value | humanizePercentage {{`}}`}} of its inodes are free. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeinodesfillingup + summary: PersistentVolumeInodes are filling up. + expr: |- + ( + kubelet_volume_stats_inodes_free{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_inodes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.15 + and + kubelet_volume_stats_inodes_used{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + and + predict_linear(kubelet_volume_stats_inodes_free{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeInodesFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeInodesFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeErrors | default false) }} + - alert: KubePersistentVolumeErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: The persistent volume {{`{{`}} $labels.persistentvolume {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} has status {{`{{`}} $labels.phase {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeerrors + summary: PersistentVolume is having issues with provisioning. + expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="{{ $kubeStateMetricsJob }}"} > 0 + for: {{ dig "KubePersistentVolumeErrors" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeErrors" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml new file mode 100644 index 0000000000..af34a23f88 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml @@ -0,0 +1,193 @@ +{{- /* +Generated from 'kubernetes-system-apiserver' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-apiserver" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-apiserver + rules: +{{- if not (.Values.defaultRules.disabled.KubeClientCertificateExpiration | default false) }} + - alert: KubeClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 7.0 days. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeclientcertificateexpiration + summary: Client certificate is about to expire. + expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job) histogram_quantile(0.01, sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 604800 + for: {{ dig "KubeClientCertificateExpiration" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeClientCertificateExpiration" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeClientCertificateExpiration | default false) }} + - alert: KubeClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 24.0 hours. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeclientcertificateexpiration + summary: Client certificate is about to expire. + expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job) histogram_quantile(0.01, sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400 + for: {{ dig "KubeClientCertificateExpiration" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeClientCertificateExpiration" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAggregatedAPIErrors | default false) }} + - alert: KubeAggregatedAPIErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubernetes aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has reported errors. It has appeared unavailable {{`{{`}} $value | humanize {{`}}`}} times averaged over the past 10m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeaggregatedapierrors + summary: Kubernetes aggregated API has reported errors. + expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}name, namespace, cluster)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[10m])) > 4 + labels: + severity: {{ dig "KubeAggregatedAPIErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAggregatedAPIDown | default false) }} + - alert: KubeAggregatedAPIDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubernetes aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has been only {{`{{`}} $value | humanize {{`}}`}}% available over the last 10m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeaggregatedapidown + summary: Kubernetes aggregated API is down. + expr: (1 - max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}name, namespace, cluster)(avg_over_time(aggregator_unavailable_apiservice{job="apiserver"}[10m]))) * 100 < 85 + for: {{ dig "KubeAggregatedAPIDown" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeAggregatedAPIDown" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if .Values.kubeApiServer.enabled }} +{{- if not (.Values.defaultRules.disabled.KubeAPIDown | default false) }} + - alert: KubeAPIDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: KubeAPI has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapidown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="apiserver"} == 1) + for: {{ dig "KubeAPIDown" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeAPIDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPITerminatedRequests | default false) }} + - alert: KubeAPITerminatedRequests + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: The kubernetes apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapiterminatedrequests + summary: The kubernetes apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. + expr: sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) / ( sum(rate(apiserver_request_total{job="apiserver"}[10m])) + sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) ) > 0.20 + for: {{ dig "KubeAPITerminatedRequests" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeAPITerminatedRequests" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml new file mode 100644 index 0000000000..205bd59800 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml @@ -0,0 +1,55 @@ +{{- /* +Generated from 'kubernetes-system-controller-manager' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeControllerManager }} +{{- if (include "exporter.kubeControllerManager.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-controller-manager" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-controller-manager + rules: +{{- if not (.Values.defaultRules.disabled.KubeControllerManagerDown | default false) }} + - alert: KubeControllerManagerDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeControllerManager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeControllerManager | indent 8 }} +{{- end }} + description: KubeControllerManager has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecontrollermanagerdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubeControllerManager.jobName" . }}"} == 1) + for: 15m + labels: + severity: {{ dig "KubeControllerManagerDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeControllerManager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeControllerManager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} + diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml new file mode 100644 index 0000000000..66b1d62001 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml @@ -0,0 +1,56 @@ +{{- /* +Generated from 'kubernetes-system-kube-proxy' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeProxy }} +{{- if (include "exporter.kubeProxy.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-kube-proxy" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-kube-proxy + rules: +{{- if not (.Values.defaultRules.disabled.KubeProxyDown | default false) }} + - alert: KubeProxyDown + annotations: + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupAnnotations.kubeProxy }} + {{- with .Values.defaultRules.additionalRuleAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupAnnotations.kubeProxy }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + description: KubeProxy has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeproxydown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubeProxy.jobName" . }}"} == 1) + for: 15m + labels: + severity: {{ dig "KubeProxyDown" "labelsSeverity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeProxy }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeProxy }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml new file mode 100644 index 0000000000..2a55676735 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml @@ -0,0 +1,379 @@ +{{- /* +Generated from 'kubernetes-system-kubelet' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-kubelet" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-kubelet + rules: +{{- if not (.Values.defaultRules.disabled.KubeNodeNotReady | default false) }} + - alert: KubeNodeNotReady + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.node {{`}}`}} has been unready for more than 15 minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubenodenotready + summary: Node is not ready. + expr: kube_node_status_condition{job="{{ $kubeStateMetricsJob }}",condition="Ready",status="true"} == 0 + for: {{ dig "KubeNodeNotReady" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeNodeNotReady" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeNodeUnreachable | default false) }} + - alert: KubeNodeUnreachable + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.node {{`}}`}} is unreachable and some workloads may be rescheduled.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubenodeunreachable + summary: Node is unreachable. + expr: (kube_node_spec_taint{job="{{ $kubeStateMetricsJob }}",key="node.kubernetes.io/unreachable",effect="NoSchedule"} unless ignoring(key,value) kube_node_spec_taint{job="{{ $kubeStateMetricsJob }}",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}) == 1 + for: {{ dig "KubeNodeUnreachable" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeNodeUnreachable" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletTooManyPods | default false) }} + - alert: KubeletTooManyPods + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet '{{`{{`}} $labels.node {{`}}`}}' is running at {{`{{`}} $value | humanizePercentage {{`}}`}} of its Pod capacity. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubelettoomanypods + summary: Kubelet is running at capacity. + expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + (kube_pod_status_phase{job="{{ $kubeStateMetricsJob }}",phase="Running"} == 1) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance,pod,namespace,cluster) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance,pod,namespace,cluster) (1, kube_pod_info{job="{{ $kubeStateMetricsJob }}"}) + ) + / + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + kube_node_status_capacity{job="{{ $kubeStateMetricsJob }}",resource="pods"} != 1 + ) > 0.95 + for: {{ dig "KubeletTooManyPods" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletTooManyPods" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeNodeReadinessFlapping | default false) }} + - alert: KubeNodeReadinessFlapping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: The readiness status of node {{`{{`}} $labels.node {{`}}`}} has changed {{`{{`}} $value {{`}}`}} times in the last 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubenodereadinessflapping + summary: Node readiness status is flapping. + expr: sum(changes(kube_node_status_condition{job="{{ $kubeStateMetricsJob }}",status="true",condition="Ready"}[15m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) > 2 + for: {{ dig "KubeNodeReadinessFlapping" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeNodeReadinessFlapping" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletPlegDurationHigh | default false) }} + - alert: KubeletPlegDurationHigh + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{`{{`}} $value {{`}}`}} seconds on node {{`{{`}} $labels.node {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletplegdurationhigh + summary: Kubelet Pod Lifecycle Event Generator is taking too long to relist. + expr: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10 + for: {{ dig "KubeletPlegDurationHigh" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletPlegDurationHigh" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletPodStartUpLatencyHigh | default false) }} + - alert: KubeletPodStartUpLatencyHigh + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet Pod startup 99th percentile latency is {{`{{`}} $value {{`}}`}} seconds on node {{`{{`}} $labels.node {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletpodstartuplatencyhigh + summary: Kubelet Pod startup latency is too high. + expr: histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"}[5m])) by (cluster, instance, le)) * on(cluster, instance) group_left(node) kubelet_node_name{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"} > 60 + for: 15m + labels: + severity: {{ dig "KubeletPodStartUpLatencyHigh" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletClientCertificateExpiration | default false) }} + - alert: KubeletClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Client certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletclientcertificateexpiration + summary: Kubelet client certificate is about to expire. + expr: kubelet_certificate_manager_client_ttl_seconds < 604800 + labels: + severity: {{ dig "KubeletClientCertificateExpiration" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletClientCertificateExpiration | default false) }} + - alert: KubeletClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Client certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletclientcertificateexpiration + summary: Kubelet client certificate is about to expire. + expr: kubelet_certificate_manager_client_ttl_seconds < 86400 + labels: + severity: {{ dig "KubeletClientCertificateExpiration" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletServerCertificateExpiration | default false) }} + - alert: KubeletServerCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Server certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletservercertificateexpiration + summary: Kubelet server certificate is about to expire. + expr: kubelet_certificate_manager_server_ttl_seconds < 604800 + labels: + severity: {{ dig "KubeletServerCertificateExpiration" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletServerCertificateExpiration | default false) }} + - alert: KubeletServerCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Server certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletservercertificateexpiration + summary: Kubelet server certificate is about to expire. + expr: kubelet_certificate_manager_server_ttl_seconds < 86400 + labels: + severity: {{ dig "KubeletServerCertificateExpiration" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletClientCertificateRenewalErrors | default false) }} + - alert: KubeletClientCertificateRenewalErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet on node {{`{{`}} $labels.node {{`}}`}} has failed to renew its client certificate ({{`{{`}} $value | humanize {{`}}`}} errors in the last 5 minutes). + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletclientcertificaterenewalerrors + summary: Kubelet has failed to renew its client certificate. + expr: increase(kubelet_certificate_manager_client_expiration_renew_errors[5m]) > 0 + for: {{ dig "KubeletClientCertificateRenewalErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletClientCertificateRenewalErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletServerCertificateRenewalErrors | default false) }} + - alert: KubeletServerCertificateRenewalErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet on node {{`{{`}} $labels.node {{`}}`}} has failed to renew its server certificate ({{`{{`}} $value | humanize {{`}}`}} errors in the last 5 minutes). + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletservercertificaterenewalerrors + summary: Kubelet has failed to renew its server certificate. + expr: increase(kubelet_server_expiration_renew_errors[5m]) > 0 + for: {{ dig "KubeletServerCertificateRenewalErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletServerCertificateRenewalErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if (include "exporter.kubelet.enabled" .)}} +{{- if not (.Values.defaultRules.disabled.KubeletDown | default false) }} + - alert: KubeletDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"} == 1) + for: 15m + labels: + severity: {{ dig "KubeletDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml new file mode 100644 index 0000000000..9890b1c959 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml @@ -0,0 +1,54 @@ +{{- /* +Generated from 'kubernetes-system-scheduler' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeScheduler.enabled .Values.defaultRules.rules.kubeSchedulerAlerting }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-scheduler" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-scheduler + rules: +{{- if .Values.kubeScheduler.enabled }} +{{- if not (.Values.defaultRules.disabled.KubeSchedulerDown | default false) }} + - alert: KubeSchedulerDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeSchedulerAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeSchedulerAlerting | indent 8 }} +{{- end }} + description: KubeScheduler has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeschedulerdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubeScheduler.jobName" . }}"} == 1) + for: 15m + labels: + severity: {{ dig "KubeSchedulerDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system.yaml new file mode 100644 index 0000000000..621326d0ad --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system.yaml @@ -0,0 +1,87 @@ +{{- /* +Generated from 'kubernetes-system' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system + rules: +{{- if not (.Values.defaultRules.disabled.KubeVersionMismatch | default false) }} + - alert: KubeVersionMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: There are {{`{{`}} $value {{`}}`}} different semantic versions of Kubernetes components running. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeversionmismatch + summary: Different semantic versions of Kubernetes components running. + expr: count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}git_version, cluster) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"git_version","$1","git_version","(v[0-9]*.[0-9]*).*"))) > 1 + for: {{ dig "KubeVersionMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeVersionMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeClientErrors | default false) }} + - alert: KubeClientErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubernetes API server client '{{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.instance {{`}}`}}' is experiencing {{`{{`}} $value | humanizePercentage {{`}}`}} errors.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeclienterrors + summary: Kubernetes API server client is experiencing errors. + expr: |- + (sum(rate(rest_client_requests_total{job="apiserver",code=~"5.."}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, job, namespace) + / + sum(rate(rest_client_requests_total{job="apiserver"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, job, namespace)) + > 0.01 + for: {{ dig "KubeClientErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeClientErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node-exporter.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node-exporter.rules.yaml new file mode 100644 index 0000000000..5d4711ae08 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node-exporter.rules.yaml @@ -0,0 +1,188 @@ +{{- /* +Generated from 'node-exporter.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/nodeExporter-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.nodeExporterRecording }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-exporter.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-exporter.rules + rules: + - expr: |- + count without (cpu, mode) ( + node_cpu_seconds_total{job="node-exporter",mode="idle"} + ) + record: instance:node_num_cpu:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - avg without (cpu) ( + sum without (mode) (rate(node_cpu_seconds_total{job="node-exporter", mode=~"idle|iowait|steal"}[5m])) + ) + record: instance:node_cpu_utilisation:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + ( + node_load1{job="node-exporter"} + / + instance:node_num_cpu:sum{job="node-exporter"} + ) + record: instance:node_load1_per_cpu:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - ( + ( + node_memory_MemAvailable_bytes{job="node-exporter"} + or + ( + node_memory_Buffers_bytes{job="node-exporter"} + + + node_memory_Cached_bytes{job="node-exporter"} + + + node_memory_MemFree_bytes{job="node-exporter"} + + + node_memory_Slab_bytes{job="node-exporter"} + ) + ) + / + node_memory_MemTotal_bytes{job="node-exporter"} + ) + record: instance:node_memory_utilisation:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) + record: instance:node_vmstat_pgmajfault:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: rate(node_disk_io_time_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) + record: instance_device:node_disk_io_time_seconds:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) + record: instance_device:node_disk_io_time_weighted_seconds:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_receive_bytes_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_receive_bytes_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_transmit_bytes_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_transmit_bytes_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_receive_drop_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_receive_drop_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_transmit_drop_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_transmit_drop_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node-exporter.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node-exporter.yaml new file mode 100644 index 0000000000..14738cedfa --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node-exporter.yaml @@ -0,0 +1,801 @@ +{{- /* +Generated from 'node-exporter' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/nodeExporter-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.nodeExporterAlerting }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-exporter" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-exporter + rules: +{{- if not (.Values.defaultRules.disabled.NodeFilesystemSpaceFillingUp | default false) }} + - alert: NodeFilesystemSpaceFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemspacefillingup + summary: Filesystem is predicted to run out of space within the next 24 hours. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 15 + and + predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemSpaceFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemSpaceFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemSpaceFillingUp | default false) }} + - alert: NodeFilesystemSpaceFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up fast. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemspacefillingup + summary: Filesystem is predicted to run out of space within the next 4 hours. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 10 + and + predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemSpaceFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemSpaceFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfSpace | default false) }} + - alert: NodeFilesystemAlmostOutOfSpace + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutofspace + summary: Filesystem has less than 5% space left. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 5 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfSpace" "for" "30m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfSpace" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfSpace | default false) }} + - alert: NodeFilesystemAlmostOutOfSpace + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutofspace + summary: Filesystem has less than 3% space left. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 3 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfSpace" "for" "30m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfSpace" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemFilesFillingUp | default false) }} + - alert: NodeFilesystemFilesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemfilesfillingup + summary: Filesystem is predicted to run out of inodes within the next 24 hours. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 40 + and + predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemFilesFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemFilesFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemFilesFillingUp | default false) }} + - alert: NodeFilesystemFilesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up fast. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemfilesfillingup + summary: Filesystem is predicted to run out of inodes within the next 4 hours. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 20 + and + predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemFilesFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemFilesFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfFiles | default false) }} + - alert: NodeFilesystemAlmostOutOfFiles + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutoffiles + summary: Filesystem has less than 5% inodes left. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 5 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfFiles" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfFiles" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfFiles | default false) }} + - alert: NodeFilesystemAlmostOutOfFiles + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutoffiles + summary: Filesystem has less than 3% inodes left. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 3 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfFiles" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfFiles" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeNetworkReceiveErrs | default false) }} + - alert: NodeNetworkReceiveErrs + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} receive errors in the last two minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodenetworkreceiveerrs + summary: Network interface is reporting many receive errors. + expr: rate(node_network_receive_errs_total{job="node-exporter"}[2m]) / rate(node_network_receive_packets_total{job="node-exporter"}[2m]) > 0.01 + for: {{ dig "NodeNetworkReceiveErrs" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeNetworkReceiveErrs" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeNetworkTransmitErrs | default false) }} + - alert: NodeNetworkTransmitErrs + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} transmit errors in the last two minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodenetworktransmiterrs + summary: Network interface is reporting many transmit errors. + expr: rate(node_network_transmit_errs_total{job="node-exporter"}[2m]) / rate(node_network_transmit_packets_total{job="node-exporter"}[2m]) > 0.01 + for: {{ dig "NodeNetworkTransmitErrs" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeNetworkTransmitErrs" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeHighNumberConntrackEntriesUsed | default false) }} + - alert: NodeHighNumberConntrackEntriesUsed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of conntrack entries are used.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodehighnumberconntrackentriesused + summary: Number of conntrack are getting close to the limit. + expr: (node_nf_conntrack_entries{job="node-exporter"} / node_nf_conntrack_entries_limit) > 0.75 + labels: + severity: {{ dig "NodeHighNumberConntrackEntriesUsed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeTextFileCollectorScrapeError | default false) }} + - alert: NodeTextFileCollectorScrapeError + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Node Exporter text file collector on {{`{{`}} $labels.instance {{`}}`}} failed to scrape. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodetextfilecollectorscrapeerror + summary: Node Exporter text file collector failed to scrape. + expr: node_textfile_scrape_error{job="node-exporter"} == 1 + labels: + severity: {{ dig "NodeTextFileCollectorScrapeError" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeClockSkewDetected | default false) }} + - alert: NodeClockSkewDetected + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Clock at {{`{{`}} $labels.instance {{`}}`}} is out of sync by more than 0.05s. Ensure NTP is configured correctly on this host. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodeclockskewdetected + summary: Clock skew detected. + expr: |- + ( + node_timex_offset_seconds{job="node-exporter"} > 0.05 + and + deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) >= 0 + ) + or + ( + node_timex_offset_seconds{job="node-exporter"} < -0.05 + and + deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) <= 0 + ) + for: {{ dig "NodeClockSkewDetected" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeClockSkewDetected" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeClockNotSynchronising | default false) }} + - alert: NodeClockNotSynchronising + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Clock at {{`{{`}} $labels.instance {{`}}`}} is not synchronising. Ensure NTP is configured on this host. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodeclocknotsynchronising + summary: Clock not synchronising. + expr: |- + min_over_time(node_timex_sync_status{job="node-exporter"}[5m]) == 0 + and + node_timex_maxerror_seconds{job="node-exporter"} >= 16 + for: {{ dig "NodeClockNotSynchronising" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeClockNotSynchronising" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeRAIDDegraded | default false) }} + - alert: NodeRAIDDegraded + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: RAID array '{{`{{`}} $labels.device {{`}}`}}' at {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/noderaiddegraded + summary: RAID Array is degraded. + expr: node_md_disks_required{job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} - ignoring (state) (node_md_disks{state="active",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}) > 0 + for: {{ dig "NodeRAIDDegraded" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeRAIDDegraded" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeRAIDDiskFailure | default false) }} + - alert: NodeRAIDDiskFailure + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: At least one device in RAID array at {{`{{`}} $labels.instance {{`}}`}} failed. Array '{{`{{`}} $labels.device {{`}}`}}' needs attention and possibly a disk swap. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/noderaiddiskfailure + summary: Failed device in RAID array. + expr: node_md_disks{state="failed",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} > 0 + labels: + severity: {{ dig "NodeRAIDDiskFailure" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFileDescriptorLimit | default false) }} + - alert: NodeFileDescriptorLimit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: File descriptors limit at {{`{{`}} $labels.instance {{`}}`}} is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefiledescriptorlimit + summary: Kernel is predicted to exhaust file descriptors limit soon. + expr: |- + ( + node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 70 + ) + for: {{ dig "NodeFileDescriptorLimit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFileDescriptorLimit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFileDescriptorLimit | default false) }} + - alert: NodeFileDescriptorLimit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: File descriptors limit at {{`{{`}} $labels.instance {{`}}`}} is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefiledescriptorlimit + summary: Kernel is predicted to exhaust file descriptors limit soon. + expr: |- + ( + node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 90 + ) + for: {{ dig "NodeFileDescriptorLimit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFileDescriptorLimit" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeCPUHighUsage | default false) }} + - alert: NodeCPUHighUsage + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'CPU usage at {{`{{`}} $labels.instance {{`}}`}} has been above 90% for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodecpuhighusage + summary: High CPU usage. + expr: sum without(mode) (avg without (cpu) (rate(node_cpu_seconds_total{job="node-exporter", mode!="idle"}[2m]))) * 100 > 90 + for: {{ dig "NodeCPUHighUsage" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeCPUHighUsage" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeSystemSaturation | default false) }} + - alert: NodeSystemSaturation + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'System load per core at {{`{{`}} $labels.instance {{`}}`}} has been above 2 for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. + + This might indicate this instance resources saturation and can cause it becoming unresponsive. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodesystemsaturation + summary: System saturated, load per core is very high. + expr: |- + node_load1{job="node-exporter"} + / count without (cpu, mode) (node_cpu_seconds_total{job="node-exporter", mode="idle"}) > 2 + for: {{ dig "NodeSystemSaturation" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeSystemSaturation" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeMemoryMajorPagesFaults | default false) }} + - alert: NodeMemoryMajorPagesFaults + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'Memory major pages are occurring at very high rate at {{`{{`}} $labels.instance {{`}}`}}, 500 major page faults per second for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. + + Please check that there is enough memory available at this instance. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodememorymajorpagesfaults + summary: Memory major page faults are occurring at very high rate. + expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) > 500 + for: {{ dig "NodeMemoryMajorPagesFaults" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeMemoryMajorPagesFaults" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeMemoryHighUtilization | default false) }} + - alert: NodeMemoryHighUtilization + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'Memory is filling up at {{`{{`}} $labels.instance {{`}}`}}, has been above 90% for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodememoryhighutilization + summary: Host is running out of memory. + expr: 100 - (node_memory_MemAvailable_bytes{job="node-exporter"} / node_memory_MemTotal_bytes{job="node-exporter"} * 100) > 90 + for: {{ dig "NodeMemoryHighUtilization" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeMemoryHighUtilization" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeDiskIOSaturation | default false) }} + - alert: NodeDiskIOSaturation + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'Disk IO queue (aqu-sq) is high on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}}, has been above 10 for the last 30 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. + + This symptom might indicate disk saturation. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodediskiosaturation + summary: Disk IO queue is high. + expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) > 10 + for: {{ dig "NodeDiskIOSaturation" "for" "30m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeDiskIOSaturation" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeSystemdServiceFailed | default false) }} + - alert: NodeSystemdServiceFailed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Systemd service {{`{{`}} $labels.name {{`}}`}} has entered failed state at {{`{{`}} $labels.instance {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodesystemdservicefailed + summary: Systemd service has entered failed state. + expr: node_systemd_unit_state{job="node-exporter", state="failed"} == 1 + for: {{ dig "NodeSystemdServiceFailed" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeSystemdServiceFailed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeBondingDegraded | default false) }} + - alert: NodeBondingDegraded + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Bonding interface {{`{{`}} $labels.master {{`}}`}} on {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more slave failures. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodebondingdegraded + summary: Bonding interface is degraded + expr: (node_bonding_slaves - node_bonding_active) != 0 + for: {{ dig "NodeBondingDegraded" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeBondingDegraded" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node-network.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node-network.yaml new file mode 100644 index 0000000000..8dc60ef66b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node-network.yaml @@ -0,0 +1,55 @@ +{{- /* +Generated from 'node-network' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.network }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-network" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-network + rules: +{{- if not (.Values.defaultRules.disabled.NodeNetworkInterfaceFlapping | default false) }} + - alert: NodeNetworkInterfaceFlapping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.network }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.network | indent 8 }} +{{- end }} + description: Network interface "{{`{{`}} $labels.device {{`}}`}}" changing its up status often on node-exporter {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/nodenetworkinterfaceflapping + summary: Network interface is often changing its status + expr: changes(node_network_up{job="node-exporter",device!~"veth.+"}[2m]) > 2 + for: {{ dig "NodeNetworkInterfaceFlapping" "for" "2m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeNetworkInterfaceFlapping" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.network }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.network }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node.rules.yaml new file mode 100644 index 0000000000..e2911b905e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/node.rules.yaml @@ -0,0 +1,109 @@ +{{- /* +Generated from 'node.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.node }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node.rules + rules: + - expr: |- + topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node, namespace, pod) ( + label_replace(kube_pod_info{job="{{ $kubeStateMetricsJob }}",node!=""}, "pod", "$1", "pod", "(.*)") + )) + record: 'node_namespace_pod:kube_pod_info:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + node_cpu_seconds_total{mode="idle",job="node-exporter"} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) + topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, node_namespace_pod:kube_pod_info:) + ) + record: node:node_num_cpu:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum( + node_memory_MemAvailable_bytes{job="node-exporter"} or + ( + node_memory_Buffers_bytes{job="node-exporter"} + + node_memory_Cached_bytes{job="node-exporter"} + + node_memory_MemFree_bytes{job="node-exporter"} + + node_memory_Slab_bytes{job="node-exporter"} + ) + ) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + record: :node_memory_MemAvailable_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + sum without (mode) ( + rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",job="node-exporter"}[5m]) + ) + ) + record: node:node_cpu_utilization:ratio_rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) ( + node:node_cpu_utilization:ratio_rate5m + ) + record: cluster:node_cpu:ratio_rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/prometheus-operator.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/prometheus-operator.yaml new file mode 100644 index 0000000000..1f288dffe3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/prometheus-operator.yaml @@ -0,0 +1,253 @@ +{{- /* +Generated from 'prometheus-operator' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/prometheusOperator-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.prometheusOperator }} +{{- $operatorJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "operator" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus-operator" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: prometheus-operator + rules: +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorListErrors | default false) }} + - alert: PrometheusOperatorListErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Errors while performing List operations in controller {{`{{`}}$labels.controller{{`}}`}} in {{`{{`}}$labels.namespace{{`}}`}} namespace. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorlisterrors + summary: Errors while performing list operations in controller. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_list_operations_failed_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[10m])) / sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_list_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[10m]))) > 0.4 + for: {{ dig "PrometheusOperatorListErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorListErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorWatchErrors | default false) }} + - alert: PrometheusOperatorWatchErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Errors while performing watch operations in controller {{`{{`}}$labels.controller{{`}}`}} in {{`{{`}}$labels.namespace{{`}}`}} namespace. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorwatcherrors + summary: Errors while performing watch operations in controller. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m])) / sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_watch_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) > 0.4 + for: {{ dig "PrometheusOperatorWatchErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorWatchErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorSyncFailed | default false) }} + - alert: PrometheusOperatorSyncFailed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Controller {{`{{`}} $labels.controller {{`}}`}} in {{`{{`}} $labels.namespace {{`}}`}} namespace fails to reconcile {{`{{`}} $value {{`}}`}} objects. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorsyncfailed + summary: Last controller reconciliation failed + expr: min_over_time(prometheus_operator_syncs{status="failed",job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusOperatorSyncFailed" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorSyncFailed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorReconcileErrors | default false) }} + - alert: PrometheusOperatorReconcileErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of reconciling operations failed for {{`{{`}} $labels.controller {{`}}`}} controller in {{`{{`}} $labels.namespace {{`}}`}} namespace.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorreconcileerrors + summary: Errors while reconciling objects. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_reconcile_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) / (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_reconcile_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) > 0.1 + for: {{ dig "PrometheusOperatorReconcileErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorReconcileErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorStatusUpdateErrors | default false) }} + - alert: PrometheusOperatorStatusUpdateErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of status update operations failed for {{`{{`}} $labels.controller {{`}}`}} controller in {{`{{`}} $labels.namespace {{`}}`}} namespace.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorstatusupdateerrors + summary: Errors while updating objects status. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_status_update_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) / (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_status_update_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) > 0.1 + for: {{ dig "PrometheusOperatorStatusUpdateErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorStatusUpdateErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorNodeLookupErrors | default false) }} + - alert: PrometheusOperatorNodeLookupErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Errors while reconciling Prometheus in {{`{{`}} $labels.namespace {{`}}`}} Namespace. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatornodelookuperrors + summary: Errors while reconciling Prometheus. + expr: rate(prometheus_operator_node_address_lookup_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0.1 + for: {{ dig "PrometheusOperatorNodeLookupErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorNodeLookupErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorNotReady | default false) }} + - alert: PrometheusOperatorNotReady + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Prometheus operator in {{`{{`}} $labels.namespace {{`}}`}} namespace isn't ready to reconcile {{`{{`}} $labels.controller {{`}}`}} resources. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatornotready + summary: Prometheus operator not ready + expr: min by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (max_over_time(prometheus_operator_ready{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) == 0) + for: {{ dig "PrometheusOperatorNotReady" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorNotReady" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorRejectedResources | default false) }} + - alert: PrometheusOperatorRejectedResources + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Prometheus operator in {{`{{`}} $labels.namespace {{`}}`}} namespace rejected {{`{{`}} printf "%0.0f" $value {{`}}`}} {{`{{`}} $labels.controller {{`}}`}}/{{`{{`}} $labels.resource {{`}}`}} resources. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorrejectedresources + summary: Resources rejected by Prometheus operator + expr: min_over_time(prometheus_operator_managed_resources{state="rejected",job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusOperatorRejectedResources" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorRejectedResources" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/prometheus.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/prometheus.yaml new file mode 100644 index 0000000000..9dfeb1f9db --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/prometheus.yaml @@ -0,0 +1,707 @@ +{{- /* +Generated from 'prometheus' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/prometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.prometheus }} +{{- $prometheusJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: prometheus + rules: +{{- if not (.Values.defaultRules.disabled.PrometheusBadConfig | default false) }} + - alert: PrometheusBadConfig + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to reload its configuration. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusbadconfig + summary: Failed Prometheus configuration reload. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(prometheus_config_last_reload_successful{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) == 0 + for: {{ dig "PrometheusBadConfig" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusBadConfig" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusSDRefreshFailure | default false) }} + - alert: PrometheusSDRefreshFailure + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to refresh SD with mechanism {{`{{`}}$labels.mechanism{{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheussdrefreshfailure + summary: Failed Prometheus SD refresh. + expr: increase(prometheus_sd_refresh_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[10m]) > 0 + for: {{ dig "PrometheusSDRefreshFailure" "for" "20m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusSDRefreshFailure" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotificationQueueRunningFull | default false) }} + - alert: PrometheusNotificationQueueRunningFull + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Alert notification queue of Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is running full. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotificationqueuerunningfull + summary: Prometheus alert notification queue predicted to run full in less than 30m. + expr: |- + # Without min_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + predict_linear(prometheus_notifications_queue_length{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m], 60 * 30) + > + min_over_time(prometheus_notifications_queue_capacity{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + for: {{ dig "PrometheusNotificationQueueRunningFull" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusNotificationQueueRunningFull" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusErrorSendingAlertsToSomeAlertmanagers | default false) }} + - alert: PrometheusErrorSendingAlertsToSomeAlertmanagers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.1f" $value {{`}}`}}% errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} to Alertmanager {{`{{`}}$labels.alertmanager{{`}}`}}.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuserrorsendingalertstosomealertmanagers + summary: Prometheus has encountered more than 1% errors sending alerts to a specific Alertmanager. + expr: |- + ( + rate(prometheus_notifications_errors_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + / + rate(prometheus_notifications_sent_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + * 100 + > 1 + for: {{ dig "PrometheusErrorSendingAlertsToSomeAlertmanagers" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusErrorSendingAlertsToSomeAlertmanagers" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotConnectedToAlertmanagers | default false) }} + - alert: PrometheusNotConnectedToAlertmanagers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is not connected to any Alertmanagers. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotconnectedtoalertmanagers + summary: Prometheus is not connected to any Alertmanagers. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(prometheus_notifications_alertmanagers_discovered{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) < 1 + for: {{ dig "PrometheusNotConnectedToAlertmanagers" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusNotConnectedToAlertmanagers" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTSDBReloadsFailing | default false) }} + - alert: PrometheusTSDBReloadsFailing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has detected {{`{{`}}$value | humanize{{`}}`}} reload failures over the last 3h. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustsdbreloadsfailing + summary: Prometheus has issues reloading blocks from disk. + expr: increase(prometheus_tsdb_reloads_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[3h]) > 0 + for: {{ dig "PrometheusTSDBReloadsFailing" "for" "4h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTSDBReloadsFailing" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTSDBCompactionsFailing | default false) }} + - alert: PrometheusTSDBCompactionsFailing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has detected {{`{{`}}$value | humanize{{`}}`}} compaction failures over the last 3h. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustsdbcompactionsfailing + summary: Prometheus has issues compacting blocks. + expr: increase(prometheus_tsdb_compactions_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[3h]) > 0 + for: {{ dig "PrometheusTSDBCompactionsFailing" "for" "4h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTSDBCompactionsFailing" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotIngestingSamples | default false) }} + - alert: PrometheusNotIngestingSamples + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is not ingesting samples. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotingestingsamples + summary: Prometheus is not ingesting samples. + expr: |- + ( + sum without(type) (rate(prometheus_tsdb_head_samples_appended_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) <= 0 + and + ( + sum without(scrape_job) (prometheus_target_metadata_cache_entries{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}) > 0 + or + sum without(rule_group) (prometheus_rule_group_rules{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}) > 0 + ) + ) + for: {{ dig "PrometheusNotIngestingSamples" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusNotIngestingSamples" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusDuplicateTimestamps | default false) }} + - alert: PrometheusDuplicateTimestamps + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is dropping {{`{{`}} printf "%.4g" $value {{`}}`}} samples/s with different values but duplicated timestamp. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusduplicatetimestamps + summary: Prometheus is dropping samples with duplicate timestamps. + expr: rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusDuplicateTimestamps" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusDuplicateTimestamps" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOutOfOrderTimestamps | default false) }} + - alert: PrometheusOutOfOrderTimestamps + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is dropping {{`{{`}} printf "%.4g" $value {{`}}`}} samples/s with timestamps arriving out of order. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusoutofordertimestamps + summary: Prometheus drops samples with out-of-order timestamps. + expr: rate(prometheus_target_scrapes_sample_out_of_order_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusOutOfOrderTimestamps" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOutOfOrderTimestamps" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteStorageFailures | default false) }} + - alert: PrometheusRemoteStorageFailures + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} failed to send {{`{{`}} printf "%.1f" $value {{`}}`}}% of the samples to {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotestoragefailures + summary: Prometheus fails to send samples to remote storage. + expr: |- + ( + (rate(prometheus_remote_storage_failed_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + / + ( + (rate(prometheus_remote_storage_failed_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + + + (rate(prometheus_remote_storage_succeeded_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + ) + ) + * 100 + > 1 + for: {{ dig "PrometheusRemoteStorageFailures" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRemoteStorageFailures" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteWriteBehind | default false) }} + - alert: PrometheusRemoteWriteBehind + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} remote write is {{`{{`}} printf "%.1f" $value {{`}}`}}s behind for {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotewritebehind + summary: Prometheus remote write is behind. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + - ignoring(remote_name, url) group_right + max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + > 120 + for: {{ dig "PrometheusRemoteWriteBehind" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRemoteWriteBehind" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteWriteDesiredShards | default false) }} + - alert: PrometheusRemoteWriteDesiredShards + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} remote write desired shards calculation wants to run {{`{{`}} $value {{`}}`}} shards for queue {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}}, which is more than the max of {{`{{`}} printf `prometheus_remote_storage_shards_max{instance="%s",job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}` $labels.instance | query | first | value {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotewritedesiredshards + summary: Prometheus remote write desired shards calculation wants to run more than configured max shards. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + max_over_time(prometheus_remote_storage_shards_desired{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + > + max_over_time(prometheus_remote_storage_shards_max{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + for: {{ dig "PrometheusRemoteWriteDesiredShards" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRemoteWriteDesiredShards" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRuleFailures | default false) }} + - alert: PrometheusRuleFailures + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to evaluate {{`{{`}} printf "%.0f" $value {{`}}`}} rules in the last 5m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusrulefailures + summary: Prometheus is failing rule evaluations. + expr: increase(prometheus_rule_evaluation_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusRuleFailures" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRuleFailures" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusMissingRuleEvaluations | default false) }} + - alert: PrometheusMissingRuleEvaluations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has missed {{`{{`}} printf "%.0f" $value {{`}}`}} rule group evaluations in the last 5m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusmissingruleevaluations + summary: Prometheus is missing rule evaluations due to slow rule group evaluation. + expr: increase(prometheus_rule_group_iterations_missed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusMissingRuleEvaluations" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusMissingRuleEvaluations" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTargetLimitHit | default false) }} + - alert: PrometheusTargetLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has dropped {{`{{`}} printf "%.0f" $value {{`}}`}} targets because the number of targets exceeded the configured target_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustargetlimithit + summary: Prometheus has dropped targets because some scrape configs have exceeded the targets limit. + expr: increase(prometheus_target_scrape_pool_exceeded_target_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusTargetLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTargetLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusLabelLimitHit | default false) }} + - alert: PrometheusLabelLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has dropped {{`{{`}} printf "%.0f" $value {{`}}`}} targets because some samples exceeded the configured label_limit, label_name_length_limit or label_value_length_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuslabellimithit + summary: Prometheus has dropped targets because some scrape configs have exceeded the labels limit. + expr: increase(prometheus_target_scrape_pool_exceeded_label_limits_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusLabelLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusLabelLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusScrapeBodySizeLimitHit | default false) }} + - alert: PrometheusScrapeBodySizeLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed {{`{{`}} printf "%.0f" $value {{`}}`}} scrapes in the last 5m because some targets exceeded the configured body_size_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusscrapebodysizelimithit + summary: Prometheus has dropped some targets that exceeded body size limit. + expr: increase(prometheus_target_scrapes_exceeded_body_size_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusScrapeBodySizeLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusScrapeBodySizeLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusScrapeSampleLimitHit | default false) }} + - alert: PrometheusScrapeSampleLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed {{`{{`}} printf "%.0f" $value {{`}}`}} scrapes in the last 5m because some targets exceeded the configured sample_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusscrapesamplelimithit + summary: Prometheus has failed scrapes that have exceeded the configured sample limit. + expr: increase(prometheus_target_scrapes_exceeded_sample_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusScrapeSampleLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusScrapeSampleLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTargetSyncFailure | default false) }} + - alert: PrometheusTargetSyncFailure + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.0f" $value {{`}}`}} targets in Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} have failed to sync because invalid configuration was supplied.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustargetsyncfailure + summary: Prometheus has failed to sync targets. + expr: increase(prometheus_target_sync_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[30m]) > 0 + for: {{ dig "PrometheusTargetSyncFailure" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTargetSyncFailure" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusHighQueryLoad | default false) }} + - alert: PrometheusHighQueryLoad + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} query API has less than 20% available capacity in its query engine for the last 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheushighqueryload + summary: Prometheus is reaching its maximum capacity serving concurrent requests. + expr: avg_over_time(prometheus_engine_queries{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) / max_over_time(prometheus_engine_queries_concurrent_max{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0.8 + for: {{ dig "PrometheusHighQueryLoad" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusHighQueryLoad" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusErrorSendingAlertsToAnyAlertmanager | default false) }} + - alert: PrometheusErrorSendingAlertsToAnyAlertmanager + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.1f" $value {{`}}`}}% minimum errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} to any Alertmanager.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuserrorsendingalertstoanyalertmanager + summary: Prometheus encounters more than 3% errors sending alerts to any Alertmanager. + expr: |- + min without (alertmanager) ( + rate(prometheus_notifications_errors_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}",alertmanager!~``}[5m]) + / + rate(prometheus_notifications_sent_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}",alertmanager!~``}[5m]) + ) + * 100 + > 3 + for: {{ dig "PrometheusErrorSendingAlertsToAnyAlertmanager" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusErrorSendingAlertsToAnyAlertmanager" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/windows.node.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/windows.node.rules.yaml new file mode 100644 index 0000000000..7c25553861 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/windows.node.rules.yaml @@ -0,0 +1,301 @@ +{{- /* +Generated from 'windows.node.rules' group from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.windowsMonitoring.enabled .Values.defaultRules.rules.windows }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "windows.node.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: windows.node.rules + rules: + - expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) ( + windows_system_system_up_time{job="windows-exporter"} + ) + record: node:windows_node:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, core) ( + windows_cpu_time_total{job="windows-exporter"} + )) + record: node:windows_node_num_cpu:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: 1 - avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(windows_cpu_time_total{job="windows-exporter",mode="idle"}[1m])) + record: :windows_node_cpu_utilisation:avg1m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + rate(windows_cpu_time_total{job="windows-exporter",mode="idle"}[1m]) + ) + record: node:windows_node_cpu_utilisation:avg1m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_memory_available_bytes{job="windows-exporter"}) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_os_visible_memory_bytes{job="windows-exporter"}) + record: ':windows_node_memory_utilisation:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_memory_available_bytes{job="windows-exporter"} + windows_memory_cache_bytes{job="windows-exporter"}) + record: :windows_node_memory_MemFreeCached_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: (windows_memory_cache_bytes{job="windows-exporter"} + windows_memory_modified_page_list_bytes{job="windows-exporter"} + windows_memory_standby_cache_core_bytes{job="windows-exporter"} + windows_memory_standby_cache_normal_priority_bytes{job="windows-exporter"} + windows_memory_standby_cache_reserve_bytes{job="windows-exporter"}) + record: node:windows_node_memory_totalCached_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_os_visible_memory_bytes{job="windows-exporter"}) + record: :windows_node_memory_MemTotal_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (windows_memory_available_bytes{job="windows-exporter"}) + ) + record: node:windows_node_memory_bytes_available:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + windows_os_visible_memory_bytes{job="windows-exporter"} + ) + record: node:windows_node_memory_bytes_total:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + (node:windows_node_memory_bytes_total:sum - node:windows_node_memory_bytes_available:sum) + / + scalar(sum(node:windows_node_memory_bytes_total:sum)) + record: node:windows_node_memory_utilisation:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: 1 - (node:windows_node_memory_bytes_available:sum / node:windows_node_memory_bytes_total:sum) + record: 'node:windows_node_memory_utilisation:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: irate(windows_memory_swap_page_operations_total{job="windows-exporter"}[5m]) + record: node:windows_node_memory_swap_io_pages:irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_logical_disk_read_seconds_total{job="windows-exporter"}[1m]) + + irate(windows_logical_disk_write_seconds_total{job="windows-exporter"}[1m]) + ) + record: :windows_node_disk_utilisation:avg_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (irate(windows_logical_disk_read_seconds_total{job="windows-exporter"}[1m]) + + irate(windows_logical_disk_write_seconds_total{job="windows-exporter"}[1m])) + ) + record: node:windows_node_disk_utilisation:avg_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,instance,volume)( + (windows_logical_disk_size_bytes{job="windows-exporter"} + - windows_logical_disk_free_bytes{job="windows-exporter"}) + / windows_logical_disk_size_bytes{job="windows-exporter"} + ) + record: 'node:windows_node_filesystem_usage:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, volume) (windows_logical_disk_free_bytes{job="windows-exporter"} / windows_logical_disk_size_bytes{job="windows-exporter"}) + record: 'node:windows_node_filesystem_avail:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_net_bytes_total{job="windows-exporter"}[1m])) + record: :windows_node_net_utilisation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (irate(windows_net_bytes_total{job="windows-exporter"}[1m])) + ) + record: node:windows_node_net_utilisation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_net_packets_received_discarded_total{job="windows-exporter"}[1m])) + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_net_packets_outbound_discarded_total{job="windows-exporter"}[1m])) + record: :windows_node_net_saturation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (irate(windows_net_packets_received_discarded_total{job="windows-exporter"}[1m]) + + irate(windows_net_packets_outbound_discarded_total{job="windows-exporter"}[1m])) + ) + record: node:windows_node_net_saturation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/windows.pod.rules.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/windows.pod.rules.yaml new file mode 100644 index 0000000000..86340b5c05 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/rules-1.14/windows.pod.rules.yaml @@ -0,0 +1,158 @@ +{{- /* +Generated from 'windows.pod.rules' group from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.windowsMonitoring.enabled .Values.defaultRules.rules.windows }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "windows.pod.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: windows.pod.rules + rules: + - expr: windows_container_available{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_pod_container_available + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_cpu_usage_seconds_total{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_total_runtime + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_memory_usage_commit_bytes{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_memory_usage + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_memory_usage_private_working_set_bytes{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_private_working_set_usage + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_network_receive_bytes_total{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_network_received_bytes_total + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_network_transmit_bytes_total{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_network_transmitted_bytes_total + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + kube_pod_container_resource_requests{resource="memory",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_memory_request + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: kube_pod_container_resource_limits{resource="memory",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_memory_limit + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + kube_pod_container_resource_requests{resource="cpu",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_cpu_cores_request + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: kube_pod_container_resource_limits{resource="cpu",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_cpu_cores_limit + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + rate(windows_container_total_runtime{}[5m]) + ) + record: namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/secret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/secret.yaml new file mode 100644 index 0000000000..e4a1e73c7b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/secret.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.thanos .Values.prometheus.prometheusSpec.thanos.objectStorageConfig}} +{{- if and .Values.prometheus.prometheusSpec.thanos.objectStorageConfig.secret (not .Values.prometheus.prometheusSpec.thanos.objectStorageConfig.existingSecret) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + object-storage-configs.yaml: {{ toYaml .Values.prometheus.prometheusSpec.thanos.objectStorageConfig.secret | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/service.yaml new file mode 100644 index 0000000000..d61b9d6ef3 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/service.yaml @@ -0,0 +1,80 @@ +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if .Values.prometheus.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + self-monitor: {{ .Values.prometheus.serviceMonitor.selfMonitor | quote }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.service.labels }} +{{ toYaml .Values.prometheus.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.service.annotations }} + annotations: +{{ toYaml .Values.prometheus.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheus.service.clusterIP }} + clusterIP: {{ .Values.prometheus.service.clusterIP }} +{{- end }} +{{- if .Values.prometheus.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheus.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheus.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheus.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheus.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheus.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheus.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheus.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.prometheus.prometheusSpec.portName }} + {{- if eq .Values.prometheus.service.type "NodePort" }} + nodePort: {{ .Values.prometheus.service.nodePort }} + {{- end }} + port: {{ .Values.prometheus.service.port }} + targetPort: {{ .Values.prometheus.service.targetPort }} + - name: reloader-web + {{- if semverCompare "> 1.20.0-0" $kubeTargetVersion }} + appProtocol: http + {{- end }} + port: {{ .Values.prometheus.service.reloaderWebPort }} + targetPort: reloader-web + {{- if .Values.prometheus.thanosIngress.enabled }} + - name: grpc + {{- if eq .Values.prometheus.service.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosIngress.nodePort }} + {{- end }} + port: {{ .Values.prometheus.thanosIngress.servicePort }} + targetPort: {{ .Values.prometheus.thanosIngress.servicePort }} + {{- end }} +{{- if .Values.prometheus.service.additionalPorts }} +{{ toYaml .Values.prometheus.service.additionalPorts | indent 2 }} +{{- end }} + publishNotReadyAddresses: {{ .Values.prometheus.service.publishNotReadyAddresses }} + selector: + {{- if .Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + {{- else }} + app.kubernetes.io/name: prometheus + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- if .Values.prometheus.service.sessionAffinity }} + sessionAffinity: {{ .Values.prometheus.service.sessionAffinity }} +{{- end }} +{{- if eq .Values.prometheus.service.sessionAffinity "ClientIP" }} + sessionAffinityConfig: + clientIP: + timeoutSeconds: {{ .Values.prometheus.service.sessionAffinityConfig.clientIP.timeoutSeconds }} +{{- end }} + type: "{{ .Values.prometheus.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceThanosSidecar.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceThanosSidecar.yaml new file mode 100644 index 0000000000..15b89c8c23 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceThanosSidecar.yaml @@ -0,0 +1,39 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosService.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-discovery + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-discovery +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosService.labels }} +{{ toYaml .Values.prometheus.thanosService.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.thanosService.annotations }} + annotations: +{{ toYaml .Values.prometheus.thanosService.annotations | indent 4 }} +{{- end }} +spec: + type: {{ .Values.prometheus.thanosService.type }} + clusterIP: {{ .Values.prometheus.thanosService.clusterIP }} +{{- if ne .Values.prometheus.thanosService.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheus.thanosService.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.prometheus.thanosService.portName }} + port: {{ .Values.prometheus.thanosService.port }} + targetPort: {{ .Values.prometheus.thanosService.targetPort }} + {{- if eq .Values.prometheus.thanosService.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosService.nodePort }} + {{- end }} + - name: {{ .Values.prometheus.thanosService.httpPortName }} + port: {{ .Values.prometheus.thanosService.httpPort }} + targetPort: {{ .Values.prometheus.thanosService.targetHttpPort }} + {{- if eq .Values.prometheus.thanosService.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosService.httpNodePort }} + {{- end }} + selector: + app.kubernetes.io/name: prometheus + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceThanosSidecarExternal.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceThanosSidecarExternal.yaml new file mode 100644 index 0000000000..453eed7f1b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceThanosSidecarExternal.yaml @@ -0,0 +1,46 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosServiceExternal.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-external + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosServiceExternal.labels }} +{{ toYaml .Values.prometheus.thanosServiceExternal.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.thanosServiceExternal.annotations }} + annotations: +{{ toYaml .Values.prometheus.thanosServiceExternal.annotations | indent 4 }} +{{- end }} +spec: + type: {{ .Values.prometheus.thanosServiceExternal.type }} +{{- if .Values.prometheus.thanosServiceExternal.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheus.thanosServiceExternal.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheus.thanosServiceExternal.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheus.thanosServiceExternal.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheus.thanosServiceExternal.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheus.thanosServiceExternal.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.prometheus.thanosServiceExternal.portName }} + port: {{ .Values.prometheus.thanosServiceExternal.port }} + targetPort: {{ .Values.prometheus.thanosServiceExternal.targetPort }} + {{- if eq .Values.prometheus.thanosServiceExternal.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosServiceExternal.nodePort }} + {{- end }} + - name: {{ .Values.prometheus.thanosServiceExternal.httpPortName }} + port: {{ .Values.prometheus.thanosServiceExternal.httpPort }} + targetPort: {{ .Values.prometheus.thanosServiceExternal.targetHttpPort }} + {{- if eq .Values.prometheus.thanosServiceExternal.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosServiceExternal.httpNodePort }} + {{- end }} + selector: + app.kubernetes.io/name: prometheus + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceaccount.yaml new file mode 100644 index 0000000000..e97b989bbd --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceaccount.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.prometheus.serviceAccount.annotations | indent 4 }} +{{- end }} +automountServiceAccountToken: {{ .Values.prometheus.serviceAccount.automountServiceAccountToken }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/servicemonitor.yaml new file mode 100644 index 0000000000..a36f3e33ca --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/servicemonitor.yaml @@ -0,0 +1,97 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.prometheus.serviceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.serviceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + release: {{ $.Release.Name | quote }} + self-monitor: "true" + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.prometheus.prometheusSpec.portName }} + {{- if .Values.prometheus.serviceMonitor.interval }} + interval: {{ .Values.prometheus.serviceMonitor.interval }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.scheme }} + scheme: {{ .Values.prometheus.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.prometheus.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.serviceMonitor.bearerTokenFile }} + {{- end }} + path: "{{ trimSuffix "/" .Values.prometheus.prometheusSpec.routePrefix }}/metrics" + metricRelabelings: + {{- if .Values.prometheus.serviceMonitor.metricRelabelings }} + {{- tpl (toYaml .Values.prometheus.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.prometheus.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + - port: reloader-web + {{- if .Values.prometheus.serviceMonitor.interval }} + interval: {{ .Values.prometheus.serviceMonitor.interval }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.scheme }} + scheme: {{ .Values.prometheus.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.prometheus.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + path: "/metrics" + {{- if .Values.prometheus.serviceMonitor.metricRelabelings }} + metricRelabelings: {{- tpl (toYaml .Values.prometheus.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.prometheus.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + {{- range .Values.prometheus.serviceMonitor.additionalEndpoints }} + - port: {{ .port }} + {{- if or $.Values.prometheus.serviceMonitor.interval .interval }} + interval: {{ default $.Values.prometheus.serviceMonitor.interval .interval }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.proxyUrl .proxyUrl }} + proxyUrl: {{ default $.Values.prometheus.serviceMonitor.proxyUrl .proxyUrl }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.scheme .scheme }} + scheme: {{ default $.Values.prometheus.serviceMonitor.scheme .scheme }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.bearerTokenFile .bearerTokenFile }} + bearerTokenFile: {{ default $.Values.prometheus.serviceMonitor.bearerTokenFile .bearerTokenFile }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.tlsConfig .tlsConfig }} + tlsConfig: {{- default $.Values.prometheus.serviceMonitor.tlsConfig .tlsConfig | toYaml | nindent 6 }} + {{- end }} + path: {{ .path }} + {{- if or $.Values.prometheus.serviceMonitor.metricRelabelings .metricRelabelings }} + metricRelabelings: {{- tpl (default $.Values.prometheus.serviceMonitor.metricRelabelings .metricRelabelings | toYaml | nindent 6) . }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.relabelings .relabelings }} + relabelings: {{- default $.Values.prometheus.serviceMonitor.relabelings .relabelings | toYaml | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/servicemonitorThanosSidecar.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/servicemonitorThanosSidecar.yaml new file mode 100644 index 0000000000..0f70aabb58 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/servicemonitorThanosSidecar.yaml @@ -0,0 +1,55 @@ +{{- if and .Values.prometheus.thanosService.enabled .Values.prometheus.thanosServiceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-sidecar + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-sidecar +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.prometheus.thanosServiceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.thanosServiceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-discovery + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.prometheus.thanosService.httpPortName }} + {{- if .Values.prometheus.thanosServiceMonitor.interval }} + interval: {{ .Values.prometheus.thanosServiceMonitor.interval }} + {{- end }} + {{- if .Values.prometheus.thanosServiceMonitor.scheme }} + scheme: {{ .Values.prometheus.thanosServiceMonitor.scheme }} + {{- end }} + {{- if .Values.prometheus.thanosServiceMonitor.tlsConfig }} + tlsConfig: {{ toYaml .Values.prometheus.thanosServiceMonitor.tlsConfig | nindent 6 }} + {{- end }} + {{- if .Values.prometheus.thanosServiceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.thanosServiceMonitor.bearerTokenFile }} + {{- end }} + path: "/metrics" + metricRelabelings: + {{- if .Values.prometheus.thanosServiceMonitor.metricRelabelings}} + {{ tpl (toYaml .Values.prometheus.thanosServiceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.prometheus.thanosServiceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.prometheus.thanosServiceMonitor.relabelings | indent 6 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/servicemonitors.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/servicemonitors.yaml new file mode 100644 index 0000000000..a7a301babc --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/servicemonitors.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.additionalServiceMonitors }} +apiVersion: v1 +kind: List +items: +{{- range .Values.prometheus.additionalServiceMonitors }} + - apiVersion: monitoring.coreos.com/v1 + kind: ServiceMonitor + metadata: + name: {{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + {{- include "servicemonitor.scrapeLimits" . | nindent 6 }} + endpoints: +{{ toYaml .endpoints | indent 8 }} + {{- if .jobLabel }} + jobLabel: {{ .jobLabel }} + {{- end }} + {{- if .namespaceSelector }} + namespaceSelector: +{{ toYaml .namespaceSelector | indent 8 }} + {{- end }} + selector: +{{ toYaml .selector | indent 8 }} + {{- if .targetLabels }} + targetLabels: +{{ toYaml .targetLabels | indent 8 }} + {{- end }} + {{- if .podTargetLabels }} + podTargetLabels: +{{ toYaml .podTargetLabels | indent 8 }} + {{- end }} + {{- if .metricRelabelings }} + metricRelabelings: +{{ toYaml .metricRelabelings | indent 8 }} + {{- end }} + {{- if .relabelings }} + relabelings: +{{ toYaml .relabelings | indent 8 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceperreplica.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceperreplica.yaml new file mode 100644 index 0000000000..4bc7f7b869 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/prometheus/serviceperreplica.yaml @@ -0,0 +1,54 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.servicePerReplica.enabled }} +{{- $count := .Values.prometheus.prometheusSpec.replicas | int -}} +{{- $serviceValues := .Values.prometheus.servicePerReplica -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-serviceperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- range $i, $e := until $count }} + - apiVersion: v1 + kind: Service + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $serviceValues.annotations }} + annotations: +{{ toYaml $serviceValues.annotations | indent 8 }} + {{- end }} + spec: + {{- if $serviceValues.clusterIP }} + clusterIP: {{ $serviceValues.clusterIP }} + {{- end }} + {{- if $serviceValues.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := $serviceValues.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} + {{- end }} + {{- if ne $serviceValues.type "ClusterIP" }} + externalTrafficPolicy: {{ $serviceValues.externalTrafficPolicy }} + {{- end }} + ports: + - name: {{ $.Values.prometheus.prometheusSpec.portName }} + {{- if eq $serviceValues.type "NodePort" }} + nodePort: {{ $serviceValues.nodePort }} + {{- end }} + port: {{ $serviceValues.port }} + targetPort: {{ $serviceValues.targetPort }} + selector: + {{- if $.Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + statefulset.kubernetes.io/pod-name: prom-agent-{{ include "kube-prometheus-stack.prometheus.crname" $ }}-{{ $i }} + {{- else }} + app.kubernetes.io/name: prometheus + statefulset.kubernetes.io/pod-name: prometheus-{{ include "kube-prometheus-stack.prometheus.crname" $ }}-{{ $i }} + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" $ }} + type: "{{ $serviceValues.type }}" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/clusterrole.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/clusterrole.yaml new file mode 100644 index 0000000000..56ca9f5eae --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/clusterrole.yaml @@ -0,0 +1,135 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-admin + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-admin: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + verbs: + - 'get' + - 'list' + - 'watch' +- apiGroups: + - monitoring.coreos.com + resources: + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + - probes + - probes/finalizers + - alertmanagerconfigs + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-edit + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-edit: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + verbs: + - 'get' + - 'list' + - 'watch' +- apiGroups: + - monitoring.coreos.com + resources: + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + - probes + - alertmanagerconfigs + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-view + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-view: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + - probes + - probes/finalizers + - alertmanagerconfigs + verbs: + - 'get' + - 'list' + - 'watch' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-ui-view + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - services/proxy + resourceNames: + - "http:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}" + - "https:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}" + - "http:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}" + - "https:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}" +{{- if .Values.grafana.enabled }} + - "http:{{ include "call-nested" (list . "grafana" "grafana.fullname") }}:{{ .Values.grafana.service.port }}" + - "https:{{ include "call-nested" (list . "grafana" "grafana.fullname") }}:{{ .Values.grafana.service.port }}" +{{- end }} + verbs: + - 'get' + - 'create' +- apiGroups: + - "" + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-prometheus + - {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +{{- if .Values.grafana.enabled }} + - {{ include "call-nested" (list . "grafana" "grafana.fullname") }} +{{- end }} + resources: + - endpoints + verbs: + - list +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/config-role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/config-role.yaml new file mode 100644 index 0000000000..f48ffc827e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/config-role.yaml @@ -0,0 +1,48 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-admin + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-edit + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-view + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - 'get' + - 'list' + - 'watch' +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboard-role.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboard-role.yaml new file mode 100644 index 0000000000..d2f81976a2 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboard-role.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create .Values.grafana.enabled }} +{{- if .Values.grafana.defaultDashboardsEnabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-admin + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-edit + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-view + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - 'get' + - 'list' + - 'watch' +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml new file mode 100644 index 0000000000..7b51a0bf7a --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.ingressNginx.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "ingress-nginx" | trunc 63 | trimSuffix "-" }} + {{- if .Values.grafana.sidecar.dashboards.annotations }} + annotations: {{ toYaml .Values.grafana.sidecar.dashboards.annotations | nindent 4 }} + {{- end }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/ingress-nginx/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml new file mode 100644 index 0000000000..d73b257451 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-cluster + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/cluster/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml new file mode 100644 index 0000000000..8865efa932 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-home + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/home/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml new file mode 100644 index 0000000000..9b05cea2e8 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-fleet-dashboards + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/fleet/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml new file mode 100644 index 0000000000..2afae10ef7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml @@ -0,0 +1,31 @@ +{{- $files := (.Files.Glob "files/rancher/k8s/*").AsConfig }} +{{- $filesDict := (fromYaml $files) }} +{{- if not (include "exporter.kubeEtcd.enabled" .) }} +{{- $filesDict = (unset $filesDict "rancher-etcd-nodes.json") -}} +{{- $filesDict = (unset $filesDict "rancher-etcd.json") -}} +{{- end }} +{{- if not (include "exporter.kubeControllerManager.enabled" .) }} +{{- $filesDict = (unset $filesDict "rancher-k8s-components-nodes.json") -}} +{{- $filesDict = (unset $filesDict "rancher-k8s-components.json") -}} +{{- else }} +{{- $_ := (set $filesDict "rancher-k8s-components-nodes.json" (get $filesDict "rancher-k8s-components-nodes.json" | replace "kube-controller-manager" (include "exporter.kubeControllerManager.jobName" .))) -}} +{{- $_ := (set $filesDict "rancher-k8s-components.json" (get $filesDict "rancher-k8s-components.json" | replace "kube-controller-manager" (include "exporter.kubeControllerManager.jobName" .))) -}} +{{- end }} +{{ $files = (toYaml $filesDict) }} +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-k8s + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ $files | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml new file mode 100644 index 0000000000..172c36e9d1 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-nodes + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/nodes/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml new file mode 100644 index 0000000000..19836ec4e4 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml @@ -0,0 +1,18 @@ +{{- $selector := (include "rancher.serviceMonitor.selector" .) -}} +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.rancherMonitoring.enabled $selector }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-performance-debugging + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/performance/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml new file mode 100644 index 0000000000..940f18869b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-pods + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/pods/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml new file mode 100644 index 0000000000..d146dacdd0 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-workloads + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/workloads/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml new file mode 100644 index 0000000000..90d24c2061 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml @@ -0,0 +1,53 @@ +{{- if .Values.rancherMonitoring.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + name: monitoring-fleet-controller + namespace: cattle-fleet-system +spec: + endpoints: + - port: metrics + metricRelabelings: + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + jobLabel: fleet + selector: + matchLabels: + app: fleet-controller +{{- end }} +--- +{{- if .Values.rancherMonitoring.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + name: monitoring-gitops-controller + namespace: cattle-fleet-system +spec: + endpoints: + - port: metrics + metricRelabelings: + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + jobLabel: gitops + selector: + matchLabels: + app: gitjob +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml new file mode 100644 index 0000000000..53a9ad6897 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml @@ -0,0 +1,27 @@ +{{- if and (not .Values.ingressNginx.enabled) (.Values.rkeIngressNginx.enabled) }} +{{- fail "Cannot set .Values.rkeIngressNginx.enabled=true when .Values.ingressNginx.enabled=false" }} +{{- end }} +{{- if and .Values.ingressNginx.enabled (not .Values.rkeIngressNginx.enabled) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-ingress-nginx + labels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx + jobLabel: ingress-nginx +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: {{ .Values.ingressNginx.namespace }} +spec: + clusterIP: None + ports: + - name: http-metrics + port: {{ .Values.ingressNginx.service.port }} + protocol: TCP + targetPort: {{ .Values.ingressNginx.service.targetPort }} + selector: + {{- if .Values.ingressNginx.service.selector }} +{{ toYaml .Values.ingressNginx.service.selector | indent 4 }} + {{- else }} + app: ingress-nginx + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml new file mode 100644 index 0000000000..b0f92e63b5 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml @@ -0,0 +1,49 @@ +{{- if and (not .Values.ingressNginx.enabled) (.Values.rkeIngressNginx.enabled) }} +{{- fail "Cannot set .Values.rkeIngressNginx.enabled=true when .Values.ingressNginx.enabled=false" }} +{{- end }} +{{- if and .Values.ingressNginx.enabled (not .Values.rkeIngressNginx.enabled) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-ingress-nginx + namespace: {{ .Values.ingressNginx.namespace }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: jobLabel + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ .Values.ingressNginx.namespace }} + endpoints: + - port: http-metrics + {{- if .Values.ingressNginx.serviceMonitor.interval}} + interval: {{ .Values.ingressNginx.serviceMonitor.interval }} + {{- end }} + {{- if .Values.ingressNginx.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.ingressNginx.serviceMonitor.proxyUrl}} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + metricRelabelings: + {{- if .Values.ingressNginx.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.ingressNginx.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.ingressNginx.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.ingressNginx.serviceMonitor.relabelings | indent 4 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml new file mode 100644 index 0000000000..1fba8f23f7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml @@ -0,0 +1,58 @@ +{{- $selector := (include "rancher.serviceMonitor.selector" .) -}} +{{- if and .Values.rancherMonitoring.enabled $selector }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + name: rancher + namespace: cattle-system +spec: + endpoints: + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + port: http + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + serverName: rancher + metricRelabelings: + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + jobLabel: rancher +{{- if .Values.rancherMonitoring.namespaceSelector }} + namespaceSelector: {{ .Values.rancherMonitoring.namespaceSelector | toYaml | nindent 4 }} +{{- end }} + selector: {{ include "rancher.serviceMonitor.selector" . | nindent 4 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-rancher-metrics +rules: +- apiGroups: + - management.cattle.io + resources: + - ranchermetrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-rancher-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-rancher-metrics +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/hardened.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/hardened.yaml new file mode 100644 index 0000000000..f9a66151ee --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/hardened.yaml @@ -0,0 +1,147 @@ +{{- $namespaces := dict "_0" .Release.Namespace -}} +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled (not .Values.grafana.defaultDashboards.useExistingNamespace) -}} +{{- $_ := set $namespaces "_1" .Values.grafana.defaultDashboards.namespace -}} +{{- end -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded, before-hook-creation +spec: + template: + metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa + spec: + serviceAccountName: {{ .Chart.Name }}-patch-sa + securityContext: + runAsNonRoot: true + runAsUser: 1000 + restartPolicy: Never + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + containers: + {{- range $_, $ns := $namespaces }} + - name: patch-sa-{{ $ns }} + image: {{ template "system_default_registry" $ }}{{ $.Values.global.kubectl.repository }}:{{ $.Values.global.kubectl.tag }} + imagePullPolicy: {{ $.Values.global.kubectl.pullPolicy }} + command: ["kubectl", "patch", "serviceaccount", "default", "-p", "{\"automountServiceAccountToken\": false}"] + args: ["-n", "{{ $ns }}"] + {{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa +rules: +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: ['get', 'patch'] +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ .Chart.Name }}-patch-sa +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Chart.Name }}-patch-sa +subjects: +- kind: ServiceAccount + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- range $_, $ns := $namespaces }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-allow-all + namespace: {{ $ns }} +spec: + podSelector: {} + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress +{{- end }} +{{- end }} +--- +{{- if .Values.hardened.k3s.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: rancher-monitoring-coredns-allow-all + namespace: kube-system +spec: + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress + podSelector: + matchLabels: + k8s-app: kube-dns +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/upgrade/configmap.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/upgrade/configmap.yaml new file mode 100644 index 0000000000..53cb898214 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/upgrade/configmap.yaml @@ -0,0 +1,13 @@ +{{- if .Values.upgrade.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "0" +data: +{{ (.Files.Glob "files/upgrade/scripts/*").AsConfig | indent 2 }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/upgrade/job.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/upgrade/job.yaml new file mode 100644 index 0000000000..8f2771740c --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/upgrade/job.yaml @@ -0,0 +1,46 @@ +{{- if .Values.upgrade.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "2" +spec: + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + spec: + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + securityContext: + runAsNonRoot: false + runAsUser: 0 + restartPolicy: Never + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + containers: + - name: run-scripts + image: {{ template "system_default_registry" . }}{{ .Values.upgrade.image.repository }}:{{ .Values.upgrade.image.tag }} + imagePullPolicy: {{ $.Values.global.kubectl.pullPolicy }} + command: + - /bin/sh + - -c + - > + for s in $(find /etc/scripts -type f); do + echo "Running $s..."; + cat $s | bash + done; + volumeMounts: + - name: upgrade + mountPath: /etc/scripts + volumes: + - name: upgrade + configMap: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/upgrade/rbac.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/upgrade/rbac.yaml new file mode 100644 index 0000000000..e929a19925 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/rancher-monitoring/upgrade/rbac.yaml @@ -0,0 +1,131 @@ +{{- if .Values.upgrade.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded + "helm.sh/hook-weight": "1" +rules: +- apiGroups: + - apps + resources: + - deployments + - daemonsets + - statefulsets + verbs: + - 'list' + - 'delete' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade +subjects: +- kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-upgrade +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade +subjects: +- kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +spec: + privileged: false + allowPrivilegeEscalation: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'configMap' + - 'secret' +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/extrasecret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/extrasecret.yaml new file mode 100644 index 0000000000..587fca2dca --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.thanosRuler.extraSecret.data -}} +{{- $secretName := printf "%s-extra" (include "kube-prometheus-stack.thanosRuler.name" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.thanosRuler.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.thanosRuler.extraSecret.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + app.kubernetes.io/component: thanos-ruler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.thanosRuler.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/ingress.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/ingress.yaml new file mode 100644 index 0000000000..e245ad448e --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/ingress.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.ingress.enabled }} +{{- $pathType := .Values.thanosRuler.ingress.pathType | default "ImplementationSpecific" }} +{{- $serviceName := include "kube-prometheus-stack.thanosRuler.name" . }} +{{- $servicePort := .Values.thanosRuler.service.port -}} +{{- $routePrefix := list .Values.thanosRuler.thanosRulerSpec.routePrefix }} +{{- $paths := .Values.thanosRuler.ingress.paths | default $routePrefix -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.thanosRuler.ingress.annotations }} + annotations: + {{- tpl (toYaml .Values.thanosRuler.ingress.annotations) . | nindent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- if .Values.thanosRuler.ingress.labels }} +{{ toYaml .Values.thanosRuler.ingress.labels | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if $apiIsStable }} + {{- if .Values.thanosRuler.ingress.ingressClassName }} + ingressClassName: {{ .Values.thanosRuler.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.thanosRuler.ingress.hosts }} + {{- range $host := .Values.thanosRuler.ingress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.thanosRuler.ingress.tls }} + tls: +{{ tpl (toYaml .Values.thanosRuler.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/podDisruptionBudget.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/podDisruptionBudget.yaml new file mode 100644 index 0000000000..83e54edf9b --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/podDisruptionBudget.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.podDisruptionBudget.enabled }} +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.thanosRuler.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.thanosRuler.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.thanosRuler.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.thanosRuler.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: thanos-ruler + thanos-ruler: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/ruler.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/ruler.yaml new file mode 100644 index 0000000000..b281221563 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/ruler.yaml @@ -0,0 +1,189 @@ +{{- if .Values.thanosRuler.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ThanosRuler +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ include "kube-prometheus-stack.thanosRuler.name" . }} +{{- include "kube-prometheus-stack.labels" . | indent 4 -}} +{{- if .Values.thanosRuler.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.thanosRuler.thanosRulerSpec.image }} + {{- $registry := include "monitoring_registry" . | default .Values.thanosRuler.thanosRulerSpec.image.registry -}} + {{- if and .Values.thanosRuler.thanosRulerSpec.image.tag .Values.thanosRuler.thanosRulerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}:{{ .Values.thanosRuler.thanosRulerSpec.image.tag }}@sha256:{{ .Values.thanosRuler.thanosRulerSpec.image.sha }}" + {{- else if .Values.thanosRuler.thanosRulerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}@sha256:{{ .Values.thanosRuler.thanosRulerSpec.image.sha }}" + {{- else if .Values.thanosRuler.thanosRulerSpec.image.tag }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}:{{ .Values.thanosRuler.thanosRulerSpec.image.tag }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}" + {{- end }} + {{- if .Values.thanosRuler.thanosRulerSpec.image.sha }} + sha: {{ .Values.thanosRuler.thanosRulerSpec.image.sha }} + {{- end }} +{{- end }} + replicas: {{ .Values.thanosRuler.thanosRulerSpec.replicas }} + listenLocal: {{ .Values.thanosRuler.thanosRulerSpec.listenLocal }} + serviceAccountName: {{ template "kube-prometheus-stack.thanosRuler.serviceAccountName" . }} +{{- if .Values.thanosRuler.thanosRulerSpec.externalPrefix }} + externalPrefix: "{{ tpl .Values.thanosRuler.thanosRulerSpec.externalPrefix . }}" +{{- else if and .Values.thanosRuler.ingress.enabled .Values.thanosRuler.ingress.hosts }} + externalPrefix: "http://{{ tpl (index .Values.thanosRuler.ingress.hosts 0) . }}{{ .Values.thanosRuler.thanosRulerSpec.routePrefix }}" +{{- else }} + externalPrefix: http://{{ template "kube-prometheus-stack.thanosRuler.name" . }}.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.thanosRuler.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.thanosRuler.thanosRulerSpec.additionalArgs }} + additionalArgs: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.additionalArgs | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.nodeSelector }} +{{ toYaml .Values.thanosRuler.thanosRulerSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.thanosRuler.thanosRulerSpec.paused }} + logFormat: {{ .Values.thanosRuler.thanosRulerSpec.logFormat | quote }} + logLevel: {{ .Values.thanosRuler.thanosRulerSpec.logLevel | quote }} + retention: {{ .Values.thanosRuler.thanosRulerSpec.retention | quote }} +{{- if .Values.thanosRuler.thanosRulerSpec.evaluationInterval }} + evaluationInterval: {{ .Values.thanosRuler.thanosRulerSpec.evaluationInterval }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.ruleNamespaceSelector }} + ruleNamespaceSelector: +{{ tpl (toYaml .Values.thanosRuler.thanosRulerSpec.ruleNamespaceSelector | indent 4) . }} +{{ else }} + ruleNamespaceSelector: {} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.ruleSelector }} + ruleSelector: +{{ tpl (toYaml .Values.thanosRuler.thanosRulerSpec.ruleSelector | indent 4) .}} +{{- else if .Values.thanosRuler.thanosRulerSpec.ruleSelectorNilUsesHelmValues }} + ruleSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + ruleSelector: {} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.alertQueryUrl }} + alertQueryUrl: "{{ .Values.thanosRuler.thanosRulerSpec.alertQueryUrl }}" +{{- end}} +{{- if .Values.thanosRuler.thanosRulerSpec.alertmanagersUrl }} + alertmanagersUrl: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.alertmanagersUrl | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.existingSecret }} + alertmanagersConfig: + key: "{{.Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.existingSecret.key }}" + name: "{{.Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.existingSecret.name }}" +{{- else if .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.secret }} + alertmanagersConfig: + key: alertmanager-configs.yaml + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.queryEndpoints }} + queryEndpoints: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.queryEndpoints | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.queryConfig.existingSecret }} + queryConfig: + key: "{{.Values.thanosRuler.thanosRulerSpec.queryConfig.existingSecret.key }}" + name: "{{.Values.thanosRuler.thanosRulerSpec.queryConfig.existingSecret.name }}" +{{- else if .Values.thanosRuler.thanosRulerSpec.queryConfig.secret }} + queryConfig: + key: query-configs.yaml + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.resources }} + resources: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.resources | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.routePrefix }} + routePrefix: "{{ .Values.thanosRuler.thanosRulerSpec.routePrefix }}" +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.securityContext }} + securityContext: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.securityContext | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.storage }} + storage: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.storage | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.objectStorageConfig.existingSecret }} + objectStorageConfig: + key: "{{.Values.thanosRuler.thanosRulerSpec.objectStorageConfig.existingSecret.key }}" + name: "{{.Values.thanosRuler.thanosRulerSpec.objectStorageConfig.existingSecret.name }}" +{{- else if .Values.thanosRuler.thanosRulerSpec.objectStorageConfig.secret }} + objectStorageConfig: + key: object-storage-configs.yaml + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.labels }} + labels: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.labels | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.podMetadata }} + podMetadata: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.podMetadata | indent 4 }} +{{- end }} +{{- if or .Values.thanosRuler.thanosRulerSpec.podAntiAffinity .Values.thanosRuler.thanosRulerSpec.affinity }} + affinity: +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.affinity }} +{{ toYaml .Values.thanosRuler.thanosRulerSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.thanosRuler.thanosRulerSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.thanosRuler.thanosRulerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [thanos-ruler]} + - {key: thanos-ruler, operator: In, values: [{{ template "kube-prometheus-stack.thanosRuler.name" . }}]} +{{- else if eq .Values.thanosRuler.thanosRulerSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.thanosRuler.thanosRulerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [thanos-ruler]} + - {key: thanos-ruler, operator: In, values: [{{ template "kube-prometheus-stack.thanosRuler.name" . }}]} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.thanosRuler.thanosRulerSpec.tolerations }} +{{ toYaml .Values.thanosRuler.thanosRulerSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.global.imagePullSecrets | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.containers }} + containers: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.containers | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.initContainers }} + initContainers: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.priorityClassName }} + priorityClassName: {{.Values.thanosRuler.thanosRulerSpec.priorityClassName }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.volumes }} + volumes: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.volumeMounts | indent 4 }} +{{- end }} + portName: {{ .Values.thanosRuler.thanosRulerSpec.portName }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/secret.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/secret.yaml new file mode 100644 index 0000000000..acab7fd9ae --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/secret.yaml @@ -0,0 +1,26 @@ +{{- if .Values.thanosRuler.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ include "kube-prometheus-stack.thanosRuler.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + {{- with .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig }} + {{- if and .secret (not .existingSecret) }} + alertmanager-configs.yaml: {{ toYaml .secret | b64enc | quote }} + {{- end }} + {{- end }} + {{- with .Values.thanosRuler.thanosRulerSpec.objectStorageConfig }} + {{- if and .secret (not .existingSecret) }} + object-storage-configs.yaml: {{ toYaml .secret | b64enc | quote }} + {{- end }} + {{- end }} + {{- with .Values.thanosRuler.thanosRulerSpec.queryConfig }} + {{- if and .secret (not .existingSecret) }} + query-configs.yaml: {{ toYaml .secret | b64enc | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/service.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/service.yaml new file mode 100644 index 0000000000..be0c844591 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/service.yaml @@ -0,0 +1,53 @@ +{{- if .Values.thanosRuler.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + self-monitor: {{ .Values.thanosRuler.serviceMonitor.selfMonitor | quote }} +{{- include "kube-prometheus-stack.labels" . | indent 4 -}} +{{- if .Values.thanosRuler.service.labels }} +{{ toYaml .Values.thanosRuler.service.labels | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.service.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.thanosRuler.service.clusterIP }} + clusterIP: {{ .Values.thanosRuler.service.clusterIP }} +{{- end }} +{{- if .Values.thanosRuler.service.externalIPs }} + externalIPs: +{{ toYaml .Values.thanosRuler.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.thanosRuler.service.loadBalancerIP }} +{{- end }} +{{- if .Values.thanosRuler.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.thanosRuler.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.thanosRuler.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.thanosRuler.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.thanosRuler.thanosRulerSpec.portName }} + {{- if eq .Values.thanosRuler.service.type "NodePort" }} + nodePort: {{ .Values.thanosRuler.service.nodePort }} + {{- end }} + port: {{ .Values.thanosRuler.service.port }} + targetPort: {{ .Values.thanosRuler.service.targetPort }} + protocol: TCP +{{- if .Values.thanosRuler.service.additionalPorts }} +{{ toYaml .Values.thanosRuler.service.additionalPorts | indent 2 }} +{{- end }} + selector: + app.kubernetes.io/name: thanos-ruler + thanos-ruler: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + type: "{{ .Values.thanosRuler.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/serviceaccount.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/serviceaccount.yaml new file mode 100644 index 0000000000..b58f1cd4df --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/serviceaccount.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + app.kubernetes.io/name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + app.kubernetes.io/component: thanos-ruler +{{- include "kube-prometheus-stack.labels" . | indent 4 -}} +{{- if .Values.thanosRuler.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.serviceAccount.annotations | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ toYaml .Values.global.imagePullSecrets | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/servicemonitor.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/servicemonitor.yaml new file mode 100644 index 0000000000..b2b138b498 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/thanos-ruler/servicemonitor.yaml @@ -0,0 +1,82 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.thanosRuler.serviceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.thanosRuler.serviceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + release: {{ $.Release.Name | quote }} + self-monitor: {{ .Values.thanosRuler.serviceMonitor.selfMonitor | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.thanosRuler.thanosRulerSpec.portName }} + {{- if .Values.thanosRuler.serviceMonitor.interval }} + interval: {{ .Values.thanosRuler.serviceMonitor.interval }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.thanosRuler.serviceMonitor.proxyUrl}} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.scheme }} + scheme: {{ .Values.thanosRuler.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.thanosRuler.serviceMonitor.bearerTokenFile }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.thanosRuler.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + path: "{{ trimSuffix "/" .Values.thanosRuler.thanosRulerSpec.routePrefix }}/metrics" + {{- if .Values.thanosRuler.serviceMonitor.metricRelabelings }} + metricRelabelings: {{- tpl (toYaml .Values.thanosRuler.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.thanosRuler.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + {{- range .Values.thanosRuler.serviceMonitor.additionalEndpoints }} + - port: {{ .port }} + {{- if or $.Values.thanosRuler.serviceMonitor.interval .interval }} + interval: {{ default $.Values.thanosRuler.serviceMonitor.interval .interval }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.proxyUrl .proxyUrl }} + proxyUrl: {{ default $.Values.thanosRuler.serviceMonitor.proxyUrl .proxyUrl }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.scheme .scheme }} + scheme: {{ default $.Values.thanosRuler.serviceMonitor.scheme .scheme }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.bearerTokenFile .bearerTokenFile }} + bearerTokenFile: {{ default $.Values.thanosRuler.serviceMonitor.bearerTokenFile .bearerTokenFile }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.tlsConfig .tlsConfig }} + tlsConfig: {{- default $.Values.thanosRuler.serviceMonitor.tlsConfig .tlsConfig | toYaml | nindent 6 }} + {{- end }} + path: {{ .path }} + {{- if or $.Values.thanosRuler.serviceMonitor.metricRelabelings .metricRelabelings }} + metricRelabelings: {{- tpl (default $.Values.thanosRuler.serviceMonitor.metricRelabelings .metricRelabelings | toYaml | nindent 6) . }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.relabelings .relabelings }} + relabelings: {{- default $.Values.thanosRuler.serviceMonitor.relabelings .relabelings | toYaml | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/validate-install-crd.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/validate-install-crd.yaml new file mode 100644 index 0000000000..6fcb8b3a69 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/validate-install-crd.yaml @@ -0,0 +1,23 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1alpha1/AlertmanagerConfig" false -}} +# {{- set $found "monitoring.coreos.com/v1/Alertmanager" false -}} +# {{- set $found "monitoring.coreos.com/v1/PodMonitor" false -}} +# {{- set $found "monitoring.coreos.com/v1/Probe" false -}} +# {{- set $found "monitoring.coreos.com/v1alpha1/PrometheusAgent" false -}} +# {{- set $found "monitoring.coreos.com/v1/Prometheus" false -}} +# {{- set $found "monitoring.coreos.com/v1/PrometheusRule" false -}} +# {{- set $found "monitoring.coreos.com/v1alpha1/ScrapeConfig" false -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- set $found "monitoring.coreos.com/v1/ThanosRuler" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install the corresponding CRD chart before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/validate-psp-install.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/validate-psp-install.yaml new file mode 100644 index 0000000000..a30c59d3b7 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/values.yaml b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/values.yaml new file mode 100644 index 0000000000..0bf19a8572 --- /dev/null +++ b/charts/rancher-monitoring/104.1.2-rc.1+up57.0.3/values.yaml @@ -0,0 +1,5431 @@ +# Default values for kube-prometheus-stack. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Rancher Monitoring Configuration + +## Configuration for prometheus-adapter +## ref: https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter +## +prometheus-adapter: + enabled: true + prometheus: + # Change this if you change the namespaceOverride or nameOverride of prometheus-operator + url: http://rancher-monitoring-prometheus.cattle-monitoring-system.svc + port: 9090 + +## RKE PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +rkeControllerManager: + enabled: false + metricsPort: 10257 # default to secure port as of k8s >= 1.22 + component: kube-controller-manager + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10011 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/controlplane: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.22" + values: + metricsPort: 10252 # default to insecure port in k8s < 1.22 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rkeScheduler: + enabled: false + metricsPort: 10259 + component: kube-scheduler + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10012 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/controlplane: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.23" + values: + metricsPort: 10251 # default to insecure port in k8s < 1.23 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rkeProxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rkeEtcd: + enabled: false + metricsPort: 2379 + component: kube-etcd + clients: + port: 10014 + https: + enabled: true + certDir: /etc/kubernetes/ssl + certFile: kube-etcd-*.pem + keyFile: kube-etcd-*-key.pem + caCertFile: kube-ca.pem + seLinuxOptions: + # Gives rkeEtcd permissions to read files in /etc/kubernetes/* + # Type is defined in https://github.com/rancher/rancher-selinux + type: rke_kubereader_t + nodeSelector: + node-role.kubernetes.io/etcd: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rkeIngressNginx: + enabled: false + metricsPort: 10254 + component: ingress-nginx + clients: + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + nodeSelector: + node-role.kubernetes.io/worker: "true" + +## k3s PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +k3sServer: + enabled: false + metricsPort: 10250 + component: k3s-server + clients: + port: 10013 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + rbac: + additionalRules: + - nonResourceURLs: ["/metrics/cadvisor"] + verbs: ["get"] + - apiGroups: [""] + resources: ["nodes/metrics"] + verbs: ["get"] + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + serviceMonitor: + endpoints: + - port: metrics + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/cadvisor + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/probes + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + +hardened: + k3s: + networkPolicy: + enabled: true + +## KubeADM PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +kubeAdmControllerManager: + enabled: false + metricsPort: 10257 + component: kube-controller-manager + clients: + port: 10011 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmScheduler: + enabled: false + metricsPort: 10259 + component: kube-scheduler + clients: + port: 10012 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmProxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmEtcd: + enabled: false + metricsPort: 2381 + component: kube-etcd + clients: + port: 10014 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +## rke2 PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +rke2ControllerManager: + enabled: false + metricsPort: 10257 # default to secure port as of k8s >= 1.22 + component: kube-controller-manager + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10011 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.22" + values: + metricsPort: 10252 # default to insecure port in k8s < 1.22 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rke2Scheduler: + enabled: false + metricsPort: 10259 # default to secure port as of k8s >= 1.22 + component: kube-scheduler + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10012 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.22" + values: + metricsPort: 10251 # default to insecure port in k8s < 1.22 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rke2Proxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rke2Etcd: + enabled: false + metricsPort: 2381 + component: kube-etcd + clients: + port: 10014 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/etcd: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rke2IngressNginx: + enabled: false + metricsPort: 10254 + component: ingress-nginx + # in the RKE2 cluster, the ingress-nginx-controller is deployed + # as a non-hostNetwork workload starting at the following versions + # - >= v1.22.12+rke2r1 < 1.23.0-0 + # - >= v1.23.9+rke2r1 < 1.24.0-0 + # - >= v1.24.3+rke2r1 < 1.25.0-0 + # - >= v1.25.0+rke2r1 + # As a result we do not need clients and proxies as we can directly create + # a service that targets the workload with the given app name + namespaceOverride: kube-system + clients: + enabled: false + proxy: + enabled: false + service: + selector: + app.kubernetes.io/name: rke2-ingress-nginx + kubeVersionOverrides: + - constraint: "< 1.21.0-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a DaemonSet with 1 pod when RKE2 version is < 1.21.0-0 + deployment: + enabled: false + proxy: + enabled: true + service: + selector: false + - constraint: ">= 1.21.0-0 < 1.22.12-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a hostNetwork Deployment with 1 pod when RKE2 version is >= 1.21.0-0 + deployment: + enabled: true + replicas: 1 + proxy: + enabled: true + service: + selector: false + - constraint: ">= 1.23.0-0 < v1.23.9-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a hostNetwork Deployment with 1 pod when RKE2 version is >= 1.20.0-0 + deployment: + enabled: true + replicas: 1 + proxy: + enabled: true + service: + selector: false + - constraint: ">= 1.24.0-0 < v1.24.3-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a hostNetwork Deployment with 1 pod when RKE2 version is >= 1.20.0-0 + deployment: + enabled: true + replicas: 1 + proxy: + enabled: true + service: + selector: false + + + +## Additional PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## + +# hardenedKubelet can only be deployed if kubelet.enabled=true +# If enabled, it replaces the ServiceMonitor deployed by the default kubelet option with a +# PushProx-based exporter that does not require a host port to be open to scrape metrics. +hardenedKubelet: + enabled: false + metricsPort: 10250 + component: kubelet + clients: + port: 10015 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + rbac: + additionalRules: + - nonResourceURLs: ["/metrics/cadvisor"] + verbs: ["get"] + - apiGroups: [""] + resources: ["nodes/metrics"] + verbs: ["get"] + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + serviceMonitor: + endpoints: + - port: metrics + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/cadvisor + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/probes + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + +# hardenedNodeExporter can only be deployed if nodeExporter.enabled=true +# If enabled, it replaces the ServiceMonitor deployed by the default nodeExporter with a +# PushProx-based exporter that does not require a host port to be open to scrape metrics. +hardenedNodeExporter: + enabled: false + metricsPort: 9796 + component: node-exporter + clients: + port: 10016 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +## Upgrades +upgrade: + ## Run upgrade scripts before an upgrade or rollback via a Job hook + enabled: true + ## Image to use to run the scripts + image: + repository: rancher/shell + tag: v0.2.1 + +## Rancher Monitoring +## + +rancherMonitoring: + enabled: true + + ## A namespaceSelector to identify the namespace to find the Rancher deployment + ## + namespaceSelector: + matchNames: + - cattle-system + + ## A selector to identify the Rancher deployment + ## If not set, the chart will try to search for the Rancher deployment in the cattle-system namespace and infer the selector values from it + ## If the Rancher deployment does not exist, no resources will be deployed. + ## + selector: {} + +## Component scraping nginx-ingress-controller +## +ingressNginx: + enabled: false + + ## The namespace to search for your nginx-ingress-controller + ## + namespace: ingress-nginx + + service: + port: 9913 + targetPort: 10254 + # selector: + # app: ingress-nginx + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "30s" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + +# Prometheus Operator Configuration + +## Provide a name in place of kube-prometheus-stack for `app:` labels +## NOTE: If you change this value, you must update the prometheus-adapter.prometheus.url +## +nameOverride: "rancher-monitoring" + +## Override the deployment namespace +## NOTE: If you change this value, you must update the prometheus-adapter.prometheus.url +## +namespaceOverride: "cattle-monitoring-system" + +## Provide a k8s version to auto dashboard import script example: kubeTargetVersionOverride: 1.26.6 +## +kubeTargetVersionOverride: "" + +## Allow kubeVersion to be overridden while creating the ingress +## +kubeVersionOverride: "" + +## Provide a name to substitute for the full names of resources +## +fullnameOverride: "" + +## Labels to apply to all resources +## +commonLabels: {} +# scmhash: abc123 +# myLabel: aakkmd + +## Install Prometheus Operator CRDs +## +crds: + enabled: true + +## custom Rules to override "for" and "severity" in defaultRules +## +customRules: {} + # AlertmanagerFailedReload: + # for: 3m + # AlertmanagerMembersInconsistent: + # for: 5m + # severity: "warning" + +## Create default rules for monitoring the cluster +## +defaultRules: + create: true + rules: + alertmanager: true + etcd: true + configReloaders: true + general: true + k8sContainerCpuUsageSecondsTotal: true + k8sContainerMemoryCache: true + k8sContainerMemoryRss: true + k8sContainerMemorySwap: true + k8sContainerResource: true + k8sContainerMemoryWorkingSetBytes: true + k8sPodOwner: true + kubeApiserverAvailability: true + kubeApiserverBurnrate: true + kubeApiserverHistogram: true + kubeApiserverSlos: true + kubeControllerManager: true + kubelet: true + kubeProxy: true + kubePrometheusGeneral: true + kubePrometheusNodeRecording: true + kubernetesApps: true + kubernetesResources: true + kubernetesStorage: true + kubernetesSystem: true + kubeSchedulerAlerting: true + kubeSchedulerRecording: true + kubeStateMetrics: true + network: true + node: true + nodeExporterAlerting: true + nodeExporterRecording: true + prometheus: true + prometheusOperator: true + windows: true + + ## Reduce app namespace alert scope + appNamespacesTarget: ".*" + + ## Set keep_firing_for for all alerts + keepFiringFor: "" + + ## Labels for default rules + labels: {} + ## Annotations for default rules + annotations: {} + + ## Additional labels for PrometheusRule alerts + additionalRuleLabels: {} + + ## Additional annotations for PrometheusRule alerts + additionalRuleAnnotations: {} + + ## Additional labels for specific PrometheusRule alert groups + additionalRuleGroupLabels: + alertmanager: {} + etcd: {} + configReloaders: {} + general: {} + k8sContainerCpuUsageSecondsTotal: {} + k8sContainerMemoryCache: {} + k8sContainerMemoryRss: {} + k8sContainerMemorySwap: {} + k8sContainerResource: {} + k8sPodOwner: {} + kubeApiserverAvailability: {} + kubeApiserverBurnrate: {} + kubeApiserverHistogram: {} + kubeApiserverSlos: {} + kubeControllerManager: {} + kubelet: {} + kubeProxy: {} + kubePrometheusGeneral: {} + kubePrometheusNodeRecording: {} + kubernetesApps: {} + kubernetesResources: {} + kubernetesStorage: {} + kubernetesSystem: {} + kubeSchedulerAlerting: {} + kubeSchedulerRecording: {} + kubeStateMetrics: {} + network: {} + node: {} + nodeExporterAlerting: {} + nodeExporterRecording: {} + prometheus: {} + prometheusOperator: {} + + ## Additional annotations for specific PrometheusRule alerts groups + additionalRuleGroupAnnotations: + alertmanager: {} + etcd: {} + configReloaders: {} + general: {} + k8sContainerCpuUsageSecondsTotal: {} + k8sContainerMemoryCache: {} + k8sContainerMemoryRss: {} + k8sContainerMemorySwap: {} + k8sContainerResource: {} + k8sPodOwner: {} + kubeApiserverAvailability: {} + kubeApiserverBurnrate: {} + kubeApiserverHistogram: {} + kubeApiserverSlos: {} + kubeControllerManager: {} + kubelet: {} + kubeProxy: {} + kubePrometheusGeneral: {} + kubePrometheusNodeRecording: {} + kubernetesApps: {} + kubernetesResources: {} + kubernetesStorage: {} + kubernetesSystem: {} + kubeSchedulerAlerting: {} + kubeSchedulerRecording: {} + kubeStateMetrics: {} + network: {} + node: {} + nodeExporterAlerting: {} + nodeExporterRecording: {} + prometheus: {} + prometheusOperator: {} + + additionalAggregationLabels: [] + + ## Prefix for runbook URLs. Use this to override the first part of the runbookURLs that is common to all rules. + runbookUrl: "https://runbooks.prometheus-operator.dev/runbooks" + + ## Disabled PrometheusRule alerts + disabled: {} + # KubeAPIDown: true + # NodeRAIDDegraded: true + +## Deprecated way to provide custom recording or alerting rules to be deployed into the cluster. +## +# additionalPrometheusRules: [] +# - name: my-rule-file +# groups: +# - name: my_group +# rules: +# - record: my_record +# expr: 100 * my_record + +## Provide custom recording or alerting rules to be deployed into the cluster. +## +additionalPrometheusRulesMap: {} +# rule-name: +# groups: +# - name: my_group +# rules: +# - record: my_record +# expr: 100 * my_record + +## +global: + cattle: + psp: + enabled: false + + systemDefaultRegistry: "" + ## Windows Monitoring + ## ref: https://github.com/rancher/charts/tree/dev-v2.5-source/packages/rancher-windows-exporter + ## + ## Deploys a DaemonSet of Prometheus exporters based on https://github.com/prometheus-community/windows_exporter. + ## Every Windows host must have a wins version of 0.1.0+ to use this chart (default as of Rancher 2.5.8). + ## To upgrade wins versions on Windows hosts, see https://github.com/rancher/wins/tree/master/charts/rancher-wins-upgrader. + ## + windows: + enabled: false + seLinux: + enabled: false + kubectl: + repository: rancher/kubectl + tag: v1.20.2 + pullPolicy: IfNotPresent + rbac: + ## Create RBAC resources for ServiceAccounts and users + ## + create: true + + userRoles: + ## Create default user ClusterRoles to allow users to interact with Prometheus CRs, ConfigMaps, and Secrets + create: true + ## Aggregate default user ClusterRoles into default k8s ClusterRoles + aggregateToDefaultRoles: true + + pspAnnotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + ## Global image registry to use if it needs to be overriden for some specific use cases (e.g local registries, custom images, ...) + ## + imageRegistry: docker.io + + ## Reference to one or more secrets to be used when pulling images + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + imagePullSecrets: [] + # - name: "image-pull-secret" + # or + # - "image-pull-secret" + +windowsMonitoring: + ## Deploys the windows-exporter and Windows-specific dashboards and rules (job name must be 'windows-exporter') + enabled: false + +## Configuration for prometheus-windows-exporter +## ref: https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-windows-exporter +## +prometheus-windows-exporter: + ## Enable ServiceMonitor and set Kubernetes label to use as a job label + ## + prometheus: + monitor: + enabled: true + jobLabel: jobLabel + + releaseLabel: true + + ## Set job label to 'windows-exporter' as required by the default Prometheus rules and Grafana dashboards + ## + podLabels: + jobLabel: windows-exporter + + ## Enable memory and container metrics as required by the default Prometheus rules and Grafana dashboards + ## + config: |- + collectors: + enabled: '[defaults],memory,container' + +## Configuration for alertmanager +## ref: https://prometheus.io/docs/alerting/alertmanager/ +## +alertmanager: + + ## Deploy alertmanager + ## + enabled: true + + ## Annotations for Alertmanager + ## + annotations: {} + + ## Api that prometheus will use to communicate with alertmanager. Possible values are v1, v2 + ## + apiVersion: v2 + + ## Service account for Alertmanager to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + annotations: {} + automountServiceAccountToken: true + + ## Configure pod disruption budgets for Alertmanager + ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget + ## This configuration is immutable once created and will require the PDB to be deleted to be changed + ## https://github.com/kubernetes/kubernetes/issues/45398 + ## + podDisruptionBudget: + enabled: false + minAvailable: 1 + maxUnavailable: "" + + ## Alertmanager configuration directives + ## ref: https://prometheus.io/docs/alerting/configuration/#configuration-file + ## https://prometheus.io/webtools/alerting/routing-tree-editor/ + ## + config: + global: + resolve_timeout: 5m + inhibit_rules: + - source_matchers: + - 'severity = critical' + target_matchers: + - 'severity =~ warning|info' + equal: + - 'namespace' + - 'alertname' + - source_matchers: + - 'severity = warning' + target_matchers: + - 'severity = info' + equal: + - 'namespace' + - 'alertname' + - source_matchers: + - 'alertname = InfoInhibitor' + target_matchers: + - 'severity = info' + equal: + - 'namespace' + - target_matchers: + - 'alertname = InfoInhibitor' + route: + group_by: ['namespace'] + group_wait: 30s + group_interval: 5m + repeat_interval: 12h + receiver: 'null' + routes: + - receiver: 'null' + matchers: + - alertname = "Watchdog" + receivers: + - name: 'null' + templates: + - '/etc/alertmanager/config/*.tmpl' + + ## Alertmanager configuration directives (as string type, preferred over the config hash map) + ## stringConfig will be used only, if tplConfig is true + ## ref: https://prometheus.io/docs/alerting/configuration/#configuration-file + ## https://prometheus.io/webtools/alerting/routing-tree-editor/ + ## + stringConfig: "" + + ## Pass the Alertmanager configuration directives through Helm's templating + ## engine. If the Alertmanager configuration contains Alertmanager templates, + ## they'll need to be properly escaped so that they are not interpreted by + ## Helm + ## ref: https://helm.sh/docs/developing_charts/#using-the-tpl-function + ## https://prometheus.io/docs/alerting/configuration/#tmpl_string + ## https://prometheus.io/docs/alerting/notifications/ + ## https://prometheus.io/docs/alerting/notification_examples/ + tplConfig: false + + ## Alertmanager template files to format alerts + ## By default, templateFiles are placed in /etc/alertmanager/config/ and if + ## they have a .tmpl file suffix will be loaded. See config.templates above + ## to change, add other suffixes. If adding other suffixes, be sure to update + ## config.templates above to include those suffixes. + ## ref: https://prometheus.io/docs/alerting/notifications/ + ## https://prometheus.io/docs/alerting/notification_examples/ + ## + + templateFiles: + rancher_defaults.tmpl: |- + {{- define "slack.rancher.text" -}} + {{ template "rancher.text_multiple" . }} + {{- end -}} + + {{- define "rancher.text_multiple" -}} + *[GROUP - Details]* + One or more alarms in this group have triggered a notification. + + {{- if gt (len .GroupLabels.Values) 0 }} + *Group Labels:* + {{- range .GroupLabels.SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- if .ExternalURL }} + *Link to AlertManager:* {{ .ExternalURL }} + {{- end }} + + {{- range .Alerts }} + {{ template "rancher.text_single" . }} + {{- end }} + {{- end -}} + + {{- define "rancher.text_single" -}} + {{- if .Labels.alertname }} + *[ALERT - {{ .Labels.alertname }}]* + {{- else }} + *[ALERT]* + {{- end }} + {{- if .Labels.severity }} + *Severity:* `{{ .Labels.severity }}` + {{- end }} + {{- if .Labels.cluster }} + *Cluster:* {{ .Labels.cluster }} + {{- end }} + {{- if .Annotations.summary }} + *Summary:* {{ .Annotations.summary }} + {{- end }} + {{- if .Annotations.message }} + *Message:* {{ .Annotations.message }} + {{- end }} + {{- if .Annotations.description }} + *Description:* {{ .Annotations.description }} + {{- end }} + {{- if .Annotations.runbook_url }} + *Runbook URL:* <{{ .Annotations.runbook_url }}|:spiral_note_pad:> + {{- end }} + {{- with .Labels }} + {{- with .Remove (stringSlice "alertname" "severity" "cluster") }} + {{- if gt (len .) 0 }} + *Additional Labels:* + {{- range .SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Annotations }} + {{- with .Remove (stringSlice "summary" "message" "description" "runbook_url") }} + {{- if gt (len .) 0 }} + *Additional Annotations:* + {{- range .SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end -}} + + ingress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + + labels: {} + + ## Override ingress to a different defined port on the service + # servicePort: 8081 + ## Override ingress to a different service then the default, this is useful if you need to + ## point to a specific instance of the alertmanager (eg kube-prometheus-stack-alertmanager-0) + # serviceName: kube-prometheus-stack-alertmanager-0 + + ## Hosts must be provided if Ingress is enabled. + ## + hosts: [] + # - alertmanager.domain.com + + ## Paths to use for ingress rules - one path should match the alertmanagerSpec.routePrefix + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Alertmanager Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: alertmanager-general-tls + # hosts: + # - alertmanager.example.com + + ## Configuration for Alertmanager secret + ## + secret: + annotations: {} + + # by default the alertmanager secret is not overwritten if it already exists + recreateIfExists: false + + ## Configuration for creating an Ingress that will map to each Alertmanager replica service + ## alertmanager.servicePerReplica must be enabled + ## + ingressPerReplica: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + + ## Final form of the hostname for each per replica ingress is + ## {{ ingressPerReplica.hostPrefix }}-{{ $replicaNumber }}.{{ ingressPerReplica.hostDomain }} + ## + ## Prefix for the per replica ingress that will have `-$replicaNumber` + ## appended to the end + hostPrefix: "" + ## Domain that will be used for the per replica ingress + hostDomain: "" + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## Secret name containing the TLS certificate for alertmanager per replica ingress + ## Secret must be manually created in the namespace + tlsSecretName: "" + + ## Separated secret for each per replica Ingress. Can be used together with cert-manager + ## + tlsSecretPerReplica: + enabled: false + ## Final form of the secret for each per replica ingress is + ## {{ tlsSecretPerReplica.prefix }}-{{ $replicaNumber }} + ## + prefix: "alertmanager" + + ## Configuration for Alertmanager service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port for Alertmanager Service to listen on + ## + port: 9093 + ## To be used with a proxy extraContainer port + ## + targetPort: 9093 + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30903 + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + + ## Additional ports to open for Alertmanager service + ## + additionalPorts: [] + # - name: oauth-proxy + # port: 8081 + # targetPort: 8081 + # - name: oauth-metrics + # port: 8082 + # targetPort: 8082 + + externalIPs: [] + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## If you want to make sure that connections from a particular client are passed to the same Pod each time + ## Accepts 'ClientIP' or 'None' + ## + sessionAffinity: None + + ## If you want to modify the ClientIP sessionAffinity timeout + ## The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP" + ## + sessionAffinityConfig: + clientIP: + timeoutSeconds: 10800 + + ## Service type + ## + type: ClusterIP + + ## Configuration for creating a separate Service for each statefulset Alertmanager replica + ## + servicePerReplica: + enabled: false + annotations: {} + + ## Port for Alertmanager Service per replica to listen on + ## + port: 9093 + + ## To be used with a proxy extraContainer port + targetPort: 9093 + + ## Port to expose on each node + ## Only used if servicePerReplica.type is 'NodePort' + ## + nodePort: 30904 + + ## Loadbalancer source IP ranges + ## Only used if servicePerReplica.type is "LoadBalancer" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Configuration for creating a ServiceMonitor for AlertManager + ## + serviceMonitor: + ## If true, a ServiceMonitor will be created for the AlertManager service. + ## + selfMonitor: true + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## Additional labels + ## + additionalLabels: {} + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## enableHttp2: Whether to enable HTTP2. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#endpoint + enableHttp2: true + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional Endpoints + ## + additionalEndpoints: [] + # - port: oauth-metrics + # path: /metrics + + ## Settings affecting alertmanagerSpec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerspec + ## + alertmanagerSpec: + ## Standard object's metadata. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata + ## Metadata Labels and Annotations gets propagated to the Alertmanager pods. + ## + podMetadata: {} + + ## Image of Alertmanager + ## + image: + repository: rancher/mirrored-prometheus-alertmanager + tag: v0.27.0 + sha: "" + + ## If true then the user will be responsible to provide a secret with alertmanager configuration + ## So when true the config part will be ignored (including templateFiles) and the one in the secret will be used + ## + useExistingSecret: false + + ## Secrets is a list of Secrets in the same namespace as the Alertmanager object, which shall be mounted into the + ## Alertmanager Pods. The Secrets are mounted into /etc/alertmanager/secrets/. + ## + secrets: [] + + ## If false then the user will opt out of automounting API credentials. + ## + automountServiceAccountToken: true + + ## ConfigMaps is a list of ConfigMaps in the same namespace as the Alertmanager object, which shall be mounted into the Alertmanager Pods. + ## The ConfigMaps are mounted into /etc/alertmanager/configmaps/. + ## + configMaps: [] + + ## ConfigSecret is the name of a Kubernetes Secret in the same namespace as the Alertmanager object, which contains configuration for + ## this Alertmanager instance. Defaults to 'alertmanager-' The secret is mounted into /etc/alertmanager/config. + ## + # configSecret: + + ## WebTLSConfig defines the TLS parameters for HTTPS + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerwebspec + web: {} + + ## AlertmanagerConfigs to be selected to merge and configure Alertmanager with. + ## + alertmanagerConfigSelector: {} + ## Example which selects all alertmanagerConfig resources + ## with label "alertconfig" with values any of "example-config" or "example-config-2" + # alertmanagerConfigSelector: + # matchExpressions: + # - key: alertconfig + # operator: In + # values: + # - example-config + # - example-config-2 + # + ## Example which selects all alertmanagerConfig resources with label "role" set to "example-config" + # alertmanagerConfigSelector: + # matchLabels: + # role: example-config + + ## Namespaces to be selected for AlertmanagerConfig discovery. If nil, only check own namespace. + ## + alertmanagerConfigNamespaceSelector: {} + ## Example which selects all namespaces + ## with label "alertmanagerconfig" with values any of "example-namespace" or "example-namespace-2" + # alertmanagerConfigNamespaceSelector: + # matchExpressions: + # - key: alertmanagerconfig + # operator: In + # values: + # - example-namespace + # - example-namespace-2 + + ## Example which selects all namespaces with label "alertmanagerconfig" set to "enabled" + # alertmanagerConfigNamespaceSelector: + # matchLabels: + # alertmanagerconfig: enabled + + ## AlermanagerConfig to be used as top level configuration + ## + alertmanagerConfiguration: {} + ## Example with select a global alertmanagerconfig + # alertmanagerConfiguration: + # name: global-alertmanager-Configuration + + ## Defines the strategy used by AlertmanagerConfig objects to match alerts. eg: + ## + alertmanagerConfigMatcherStrategy: {} + ## Example with use OnNamespace strategy + # alertmanagerConfigMatcherStrategy: + # type: OnNamespace + + ## Define Log Format + # Use logfmt (default) or json logging + logFormat: logfmt + + ## Log level for Alertmanager to be configured with. + ## + logLevel: info + + ## Size is the expected size of the alertmanager cluster. The controller will eventually make the size of the + ## running cluster equal to the expected size. + replicas: 1 + + ## Time duration Alertmanager shall retain data for. Default is '120h', and must match the regular expression + ## [0-9]+(ms|s|m|h) (milliseconds seconds minutes hours). + ## + retention: 120h + + ## Storage is the definition of how storage will be used by the Alertmanager instances. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/storage.md + ## + storage: {} + # volumeClaimTemplate: + # spec: + # storageClassName: gluster + # accessModes: ["ReadWriteOnce"] + # resources: + # requests: + # storage: 50Gi + # selector: {} + + + ## The external URL the Alertmanager instances will be available under. This is necessary to generate correct URLs. This is necessary if Alertmanager is not served from root of a DNS name. string false + ## + externalUrl: + + ## The route prefix Alertmanager registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, + ## but the server serves requests under a different route prefix. For example for use with kubectl proxy. + ## + routePrefix: / + + ## scheme: HTTP scheme to use. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when connect to the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + ## If set to true all actions on the underlying managed objects are not going to be performed, except for delete actions. + ## + paused: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Define resources requests and limits for single Pods. + ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + limits: + memory: 500Mi + cpu: 1000m + requests: + memory: 100Mi + cpu: 100m + + ## Pod anti-affinity can prevent the scheduler from placing Prometheus replicas on the same node. + ## The default value "soft" means that the scheduler should *prefer* to not schedule two replica pods onto the same node but no guarantee is provided. + ## The value "hard" means that the scheduler is *required* to not schedule two replica pods onto the same node. + ## The value "" will disable pod anti-affinity so that no anti-affinity rules will be configured. + ## + podAntiAffinity: "" + + ## If anti-affinity is enabled sets the topologyKey to use for anti-affinity. + ## This can be changed to, for example, failure-domain.beta.kubernetes.io/zone + ## + podAntiAffinityTopologyKey: kubernetes.io/hostname + + ## Assign custom affinity rules to the alertmanager instance + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + + ## If specified, the pod's tolerations. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## If specified, the pod's topology spread constraints. + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app: alertmanager + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 1000 and gid 2000. *v1.PodSecurityContext false + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + + ## ListenLocal makes the Alertmanager server listen on loopback, so that it does not bind against the Pod IP. + ## Note this is only for the Alertmanager UI, not the gossip communication. + ## + listenLocal: false + + ## Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to an Alertmanager pod. + ## + containers: [] + # containers: + # - name: oauth-proxy + # image: quay.io/oauth2-proxy/oauth2-proxy:v7.5.1 + # args: + # - --upstream=http://127.0.0.1:9093 + # - --http-address=0.0.0.0:8081 + # - --metrics-address=0.0.0.0:8082 + # - ... + # ports: + # - containerPort: 8081 + # name: oauth-proxy + # protocol: TCP + # - containerPort: 8082 + # name: oauth-metrics + # protocol: TCP + # resources: {} + + # Additional volumes on the output StatefulSet definition. + volumes: [] + + # Additional VolumeMounts on the output StatefulSet definition. + volumeMounts: [] + + ## InitContainers allows injecting additional initContainers. This is meant to allow doing some changes + ## (permissions, dir tree) on mounted volumes before starting prometheus + initContainers: [] + + ## Priority class assigned to the Pods + ## + priorityClassName: "" + + ## AdditionalPeers allows injecting a set of additional Alertmanagers to peer with to form a highly available cluster. + ## + additionalPeers: [] + + ## PortName to use for Alert Manager. + ## + portName: "http-web" + + ## ClusterAdvertiseAddress is the explicit address to advertise in cluster. Needs to be provided for non RFC1918 [1] (public) addresses. [1] RFC1918: https://tools.ietf.org/html/rfc1918 + ## + clusterAdvertiseAddress: false + + ## clusterGossipInterval determines interval between gossip attempts. + ## Needs to be specified as GoDuration, a time duration that can be parsed by Go’s time.ParseDuration() (e.g. 45ms, 30s, 1m, 1h20m15s) + clusterGossipInterval: "" + + ## clusterPeerTimeout determines timeout for cluster peering. + ## Needs to be specified as GoDuration, a time duration that can be parsed by Go’s time.ParseDuration() (e.g. 45ms, 30s, 1m, 1h20m15s) + clusterPeerTimeout: "" + + ## clusterPushpullInterval determines interval between pushpull attempts. + ## Needs to be specified as GoDuration, a time duration that can be parsed by Go’s time.ParseDuration() (e.g. 45ms, 30s, 1m, 1h20m15s) + clusterPushpullInterval: "" + + ## ForceEnableClusterMode ensures Alertmanager does not deactivate the cluster mode when running with a single replica. + ## Use case is e.g. spanning an Alertmanager cluster across Kubernetes clusters with a single replica in each. + forceEnableClusterMode: false + + ## Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to + ## be considered available. Defaults to 0 (pod will be considered available as soon as it is ready). + minReadySeconds: 0 + + ## Additional configuration which is not covered by the properties above. (passed through tpl) + additionalConfig: {} + + ## Additional configuration which is not covered by the properties above. + ## Useful, if you need advanced templating inside alertmanagerSpec. + ## Otherwise, use alertmanager.alertmanagerSpec.additionalConfig (passed through tpl) + additionalConfigString: "" + + ## ExtraSecret can be used to store various data in an extra secret + ## (use it for example to store hashed basic auth credentials) + extraSecret: + ## if not set, name will be auto generated + # name: "" + annotations: {} + data: {} + # auth: | + # foo:$apr1$OFG3Xybp$ckL0FHDAkoXYIlH9.cysT0 + # someoneelse:$apr1$DMZX2Z4q$6SbQIfyuLQd.xmo/P0m2c. + +## Using default values from https://github.com/grafana/helm-charts/blob/main/charts/grafana/values.yaml +## +grafana: + enabled: true + namespaceOverride: "" + + ## Grafana's primary configuration + ## NOTE: values in map will be converted to ini format + ## ref: http://docs.grafana.org/installation/configuration/ + ## + grafana.ini: + users: + auto_assign_org_role: Viewer + auth: + disable_login_form: false + auth.anonymous: + enabled: true + org_role: Viewer + auth.basic: + enabled: false + dashboards: + # Modify this value to change the default dashboard shown on the main Grafana page + default_home_dashboard_path: /tmp/dashboards/rancher-default-home.json + security: + # Required to embed dashboards in Rancher Cluster Overview Dashboard on Cluster Explorer + allow_embedding: true + + deploymentStrategy: + type: Recreate + + ## ForceDeployDatasources Create datasource configmap even if grafana deployment has been disabled + ## + forceDeployDatasources: false + + ## ForceDeployDashboard Create dashboard configmap even if grafana deployment has been disabled + ## + forceDeployDashboards: false + + ## Deploy default dashboards + ## + defaultDashboardsEnabled: true + + # Additional options for defaultDashboards + defaultDashboards: + # The default namespace to place defaultDashboards within + namespace: cattle-dashboards + # Whether to create the default namespace as a Helm managed namespace or use an existing namespace + # If false, the defaultDashboards.namespace will be created as a Helm managed namespace + useExistingNamespace: false + # Whether the Helm managed namespace created by this chart should be left behind on a Helm uninstall + # If you place other dashboards in this namespace, then they will be deleted on a helm uninstall + # Ignore if useExistingNamespace is true + cleanupOnUninstall: false + + ## Timezone for the default dashboards + ## Other options are: browser or a specific timezone, i.e. Europe/Luxembourg + ## + defaultDashboardsTimezone: utc + + ## Editable flag for the default dashboards + ## + defaultDashboardsEditable: true + + adminPassword: prom-operator + + ingress: + ## If true, Grafana Ingress will be created + ## + enabled: false + + ## IngressClassName for Grafana Ingress. + ## Should be provided if Ingress is enable. + ## + # ingressClassName: nginx + + ## Annotations for Grafana Ingress + ## + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + + ## Labels to be added to the Ingress + ## + labels: {} + + ## Hostnames. + ## Must be provided if Ingress is enable. + ## + # hosts: + # - grafana.domain.com + hosts: [] + + ## Path for grafana ingress + path: / + + ## TLS configuration for grafana Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: grafana-general-tls + # hosts: + # - grafana.example.com + + # # To make Grafana persistent (Using Statefulset) + # # + # persistence: + # enabled: true + # type: sts + # storageClassName: "storageClassName" + # accessModes: + # - ReadWriteOnce + # size: 20Gi + # finalizers: + # - kubernetes.io/pvc-protection + + serviceAccount: + create: true + autoMount: true + + sidecar: + dashboards: + enabled: true + label: grafana_dashboard + searchNamespace: cattle-dashboards + labelValue: "1" + + # Support for new table panels, when enabled grafana auto migrates the old table panels to newer table panels + enableNewTablePanelSyntax: false + + ## Annotations for Grafana dashboard configmaps + ## + annotations: {} + multicluster: + global: + enabled: false + etcd: + enabled: false + provider: + allowUiUpdates: false + datasources: + enabled: true + defaultDatasourceEnabled: true + isDefaultDatasource: true + + uid: prometheus + + ## URL of prometheus datasource + ## + # url: http://prometheus-stack-prometheus:9090/ + + ## Prometheus request timeout in seconds + # timeout: 30 + + # If not defined, will use prometheus.prometheusSpec.scrapeInterval or its default + # defaultDatasourceScrapeInterval: 15s + + ## Annotations for Grafana datasource configmaps + ## + annotations: {} + + ## Set method for HTTP to send query to datasource + httpMethod: POST + + ## Create datasource for each Pod of Prometheus StatefulSet; + ## this uses headless service `prometheus-operated` which is + ## created by Prometheus Operator + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/0fee93e12dc7c2ea1218f19ae25ec6b893460590/pkg/prometheus/statefulset.go#L255-L286 + createPrometheusReplicasDatasources: false + label: grafana_datasource + labelValue: "1" + + ## Field with internal link pointing to existing data source in Grafana. + ## Can be provisioned via additionalDataSources + exemplarTraceIdDestinations: {} + # datasourceUid: Jaeger + # traceIdLabelName: trace_id + alertmanager: + enabled: true + uid: alertmanager + handleGrafanaManagedAlerts: false + implementation: prometheus + + extraConfigmapMounts: [] + # - name: certs-configmap + # mountPath: /etc/grafana/ssl/ + # configMap: certs-configmap + # readOnly: true + + deleteDatasources: [] + # - name: example-datasource + # orgId: 1 + + ## Configure additional grafana datasources (passed through tpl) + ## ref: http://docs.grafana.org/administration/provisioning/#datasources + additionalDataSources: [] + # - name: prometheus-sample + # access: proxy + # basicAuth: true + # basicAuthPassword: pass + # basicAuthUser: daco + # editable: false + # jsonData: + # tlsSkipVerify: true + # orgId: 1 + # type: prometheus + # url: https://{{ printf "%s-prometheus.svc" .Release.Name }}:9090 + # version: 1 + + ## Passed to grafana subchart and used by servicemonitor below + ## + service: + portName: nginx-http + ## Port for Grafana Service to listen on + ## + port: 80 + ## To be used with a proxy extraContainer port + ## + targetPort: 8080 + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30950 + ## Service type + ## + type: ClusterIP + + proxy: + image: + repository: rancher/mirrored-library-nginx + tag: 1.24.0-alpine + + ## Enable an Specify container in extraContainers. This is meant to allow adding an authentication proxy to a grafana pod + extraContainers: | + - name: grafana-proxy + args: + - nginx + - -g + - daemon off; + - -c + - /nginx/nginx.conf + image: "{{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }}" + ports: + - containerPort: 8080 + name: nginx-http + protocol: TCP + volumeMounts: + - mountPath: /nginx + name: grafana-nginx + - mountPath: /var/cache/nginx + name: nginx-home + securityContext: + runAsUser: 101 + runAsGroup: 101 + + ## Volumes that can be used in containers + extraContainerVolumes: + - name: nginx-home + emptyDir: {} + - name: grafana-nginx + configMap: + name: grafana-nginx-proxy-config + items: + - key: nginx.conf + mode: 438 + path: nginx.conf + + ## If true, create a serviceMonitor for grafana + ## + serviceMonitor: + # If true, a ServiceMonitor CRD is created for a prometheus operator + # https://github.com/coreos/prometheus-operator + # + enabled: true + + # Path to use for scraping metrics. Might be different if server.root_url is set + # in grafana.ini + path: "/metrics" + + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + + # labels for the ServiceMonitor + labels: {} + + # Scrape interval. If not set, the Prometheus default scrape interval is used. + # + interval: "" + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + resources: + limits: + memory: 200Mi + cpu: 200m + requests: + memory: 100Mi + cpu: 100m + + testFramework: + enabled: false + +## Flag to disable all the kubernetes component scrapers +## +kubernetesServiceMonitors: + enabled: true + +## Component scraping the kube api server +## +kubeApiServer: + enabled: true + tlsConfig: + serverName: kubernetes + insecureSkipVerify: false + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + jobLabel: component + selector: + matchLabels: + component: apiserver + provider: kubernetes + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: + # Drop excessively noisy apiserver buckets. + - action: drop + regex: apiserver_request_duration_seconds_bucket;(0.15|0.2|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2|3|3.5|4|4.5|6|7|8|9|15|25|40|50) + sourceLabels: + - __name__ + - le + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: + # - __meta_kubernetes_namespace + # - __meta_kubernetes_service_name + # - __meta_kubernetes_endpoint_port_name + # action: keep + # regex: default;kubernetes;https + # - targetLabel: __address__ + # replacement: kubernetes.default.svc:443 + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping the kubelet and kubelet-hosted cAdvisor +## +kubelet: + enabled: true + namespace: kube-system + + serviceMonitor: + ## Attach metadata to discovered targets. Requires Prometheus v2.45 for endpoints created by the operator. + ## + attachMetadata: + node: false + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## If true, Prometheus use (respect) labels provided by exporter. + ## + honorLabels: true + + ## If true, Prometheus ingests metrics with timestamp provided by exporter. If false, Prometheus ingests metrics with timestamp of scrape. + ## + honorTimestamps: true + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## Enable scraping the kubelet over https. For requirements to enable this see + ## https://github.com/prometheus-operator/prometheus-operator/issues/926 + ## + https: true + + ## Enable scraping /metrics/cadvisor from kubelet's service + ## + cAdvisor: true + + ## Enable scraping /metrics/probes from kubelet's service + ## + probes: true + + ## Enable scraping /metrics/resource from kubelet's service + ## This is disabled by default because container metrics are already exposed by cAdvisor + ## + resource: false + # From kubernetes 1.18, /metrics/resource/v1alpha1 renamed to /metrics/resource + resourcePath: "/metrics/resource/v1alpha1" + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + cAdvisorMetricRelabelings: + # Drop less useful container CPU metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_cpu_(cfs_throttled_seconds_total|load_average_10s|system_seconds_total|user_seconds_total)' + # Drop less useful container / always zero filesystem metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_fs_(io_current|io_time_seconds_total|io_time_weighted_seconds_total|reads_merged_total|sector_reads_total|sector_writes_total|writes_merged_total)' + # Drop less useful / always zero container memory metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_memory_(mapped_file|swap)' + # Drop less useful container process metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_(file_descriptors|tasks_state|threads_max)' + # Drop container spec metrics that overlap with kube-state-metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_spec.*' + # Drop cgroup metrics with no pod. + - sourceLabels: [id, pod] + action: drop + regex: '.+;' + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + probesMetricRelabelings: [] + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + ## metrics_path is required to match upstream rules and charts + cAdvisorRelabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + probesRelabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + resourceRelabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + ## metrics_path is required to match upstream rules and charts + relabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping the kube controller manager +## +kubeControllerManager: + enabled: false + + ## If your kube controller manager is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## If using kubeControllerManager.endpoints only the port and targetPort are used + ## + service: + enabled: true + ## If null or unset, the value is determined dynamically based on target Kubernetes version due to change + ## of default port in Kubernetes 1.22. + ## + port: null + targetPort: null + # selector: + # component: kube-controller-manager + + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # component: kube-controller-manager + + ## Enable scraping kube-controller-manager over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks. + ## If null or unset, the value is determined dynamically based on target Kubernetes version. + ## + https: null + + # Skip TLS certificate validation when scraping + insecureSkipVerify: null + + # Name of the server to use when validating TLS certificate + serverName: null + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping coreDns. Use either this or kubeDns +## +coreDns: + enabled: true + service: + enabled: true + port: 9153 + targetPort: 9153 + # selector: + # k8s-app: kube-dns + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # k8s-app: kube-dns + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kubeDns. Use either this or coreDns +## +kubeDns: + enabled: false + service: + dnsmasq: + port: 10054 + targetPort: 10054 + skydns: + port: 10055 + targetPort: 10055 + # selector: + # k8s-app: kube-dns + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + jobLabel: jobLabel + selector: {} + # matchLabels: + # k8s-app: kube-dns + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + dnsmasqMetricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + dnsmasqRelabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping etcd +## +kubeEtcd: + enabled: false + + ## If your etcd is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## Etcd service. If using kubeEtcd.endpoints only the port and targetPort are used + ## + service: + enabled: true + port: 2381 + targetPort: 2381 + # selector: + # component: etcd + + ## Configure secure access to the etcd cluster by loading a secret into prometheus and + ## specifying security configuration below. For example, with a secret named etcd-client-cert + ## + ## serviceMonitor: + ## scheme: https + ## insecureSkipVerify: false + ## serverName: localhost + ## caFile: /etc/prometheus/secrets/etcd-client-cert/etcd-ca + ## certFile: /etc/prometheus/secrets/etcd-client-cert/etcd-client + ## keyFile: /etc/prometheus/secrets/etcd-client-cert/etcd-client-key + ## + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + scheme: http + insecureSkipVerify: false + serverName: "" + caFile: "" + certFile: "" + keyFile: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # component: etcd + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kube scheduler +## +kubeScheduler: + enabled: false + + ## If your kube scheduler is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## If using kubeScheduler.endpoints only the port and targetPort are used + ## + service: + enabled: true + ## If null or unset, the value is determined dynamically based on target Kubernetes version due to change + ## of default port in Kubernetes 1.23. + ## + port: null + targetPort: null + # selector: + # component: kube-scheduler + + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + ## Enable scraping kube-scheduler over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks. + ## If null or unset, the value is determined dynamically based on target Kubernetes version. + ## + https: null + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # component: kube-scheduler + + ## Skip TLS certificate validation when scraping + insecureSkipVerify: null + + ## Name of the server to use when validating TLS certificate + serverName: null + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kube proxy +## +kubeProxy: + enabled: false + + ## If your kube proxy is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + service: + enabled: true + port: 10249 + targetPort: 10249 + # selector: + # k8s-app: kube-proxy + + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # k8s-app: kube-proxy + + ## Enable scraping kube-proxy over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks + ## + https: false + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kube state metrics +## +kubeStateMetrics: + enabled: true + +## Configuration for kube-state-metrics subchart +## +kube-state-metrics: + namespaceOverride: "" + rbac: + create: true + releaseLabel: true + prometheus: + monitor: + enabled: true + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## Scrape Timeout. If not set, the Prometheus default scrape timeout is used. + ## + scrapeTimeout: "" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + # Keep labels from scraped data, overriding server-side labels + ## + honorLabels: true + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + selfMonitor: + enabled: false + +## Deploy node exporter as a daemonset to all nodes +## +nodeExporter: + enabled: true + operatingSystems: + linux: + enabled: true + darwin: + enabled: true + + ## ForceDeployDashboard Create dashboard configmap even if nodeExporter deployment has been disabled + ## + forceDeployDashboards: false + +## Configuration for prometheus-node-exporter subchart +## +prometheus-node-exporter: + namespaceOverride: "" + podLabels: + ## Add the 'node-exporter' label to be used by serviceMonitor to match standard common usage in rules and grafana dashboards + ## + jobLabel: node-exporter + releaseLabel: true + extraArgs: + - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/.+)($|/) + - --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ + service: + portName: http-metrics + prometheus: + monitor: + enabled: true + + jobLabel: jobLabel + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## How long until a scrape request times out. If not set, the Prometheus default scape timeout is used. + ## + scrapeTimeout: "" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - sourceLabels: [__name__] + # separator: ; + # regex: ^node_mountstats_nfs_(event|operations|transport)_.+ + # replacement: $1 + # action: drop + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + +## Manages Prometheus and Alertmanager components +## +prometheusOperator: + enabled: true + + ## Use '{{ template "kube-prometheus-stack.fullname" . }}-operator' by default + fullnameOverride: "" + + ## Number of old replicasets to retain ## + ## The default value is 10, 0 will garbage-collect old replicasets ## + revisionHistoryLimit: 10 + + ## Strategy of the deployment + ## + strategy: {} + + ## Prometheus-Operator v0.39.0 and later support TLS natively. + ## + tls: + enabled: true + # Value must match version names from https://golang.org/pkg/crypto/tls/#pkg-constants + tlsMinVersion: VersionTLS13 + # Users who are deploying this chart in GKE private clusters will need to add firewall rules to expose this port for admissions webhooks + internalPort: 8443 + + ## Admission webhook support for PrometheusRules resources added in Prometheus Operator 0.30 can be enabled to prevent incorrectly formatted + ## rules from making their way into prometheus and potentially preventing the container from starting + admissionWebhooks: + ## Valid values: Fail, Ignore, IgnoreOnInstallOnly + ## IgnoreOnInstallOnly - If Release.IsInstall returns "true", set "Ignore" otherwise "Fail" + failurePolicy: "" + ## The default timeoutSeconds is 10 and the maximum value is 30. + timeoutSeconds: 10 + enabled: true + ## A PEM encoded CA bundle which will be used to validate the webhook's server certificate. + ## If unspecified, system trust roots on the apiserver are used. + caBundle: "" + ## If enabled, generate a self-signed certificate, then patch the webhook configurations with the generated data. + ## On chart upgrades (or if the secret exists) the cert will not be re-generated. You can use this to provide your own + ## certs ahead of time if you wish. + ## + annotations: {} + # argocd.argoproj.io/hook: PreSync + # argocd.argoproj.io/hook-delete-policy: HookSucceeded + + namespaceSelector: {} + + deployment: + enabled: false + + ## Number of replicas + ## + replicas: 1 + + ## Strategy of the deployment + ## + strategy: {} + + # Ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ + podDisruptionBudget: {} + # maxUnavailable: 1 + # minAvailable: 1 + + ## Number of old replicasets to retain ## + ## The default value is 10, 0 will garbage-collect old replicasets ## + revisionHistoryLimit: 10 + + ## Prometheus-Operator v0.39.0 and later support TLS natively. + ## + tls: + enabled: true + # Value must match version names from https://golang.org/pkg/crypto/tls/#pkg-constants + tlsMinVersion: VersionTLS13 + # The default webhook port is 10250 in order to work out-of-the-box in GKE private clusters and avoid adding firewall rules. + internalPort: 10250 + + ## Service account for Prometheus Operator Webhook to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + automountServiceAccountToken: false + create: true + name: "" + + ## Configuration for Prometheus operator Webhook service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 31080 + + nodePortTls: 31443 + + ## Additional ports to open for Prometheus operator Webhook service + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#multi-port-services + ## + additionalPorts: [] + + ## Loadbalancer IP + ## Only use if service.type is "LoadBalancer" + ## + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## NodePort, ClusterIP, LoadBalancer + ## + type: ClusterIP + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + # ## Labels to add to the operator webhook deployment + # ## + labels: {} + + ## Annotations to add to the operator webhook deployment + ## + annotations: {} + + ## Labels to add to the operator webhook pod + ## + podLabels: {} + + ## Annotations to add to the operator webhook pod + ## + podAnnotations: {} + + ## Assign a PriorityClassName to pods if set + # priorityClassName: "" + + ## Define Log Format + # Use logfmt (default) or json logging + # logFormat: logfmt + + ## Decrease log verbosity to errors only + # logLevel: error + + ## Prometheus-operator webhook image + ## + image: + registry: quay.io + repository: rancher/mirrored-prometheus-operator-admission-webhook + # if not set appVersion field from Chart.yaml is used + tag: v0.72.0 + sha: "" + pullPolicy: IfNotPresent + + ## Define Log Format + # Use logfmt (default) or json logging + # logFormat: logfmt + + ## Decrease log verbosity to errors only + # logLevel: error + + + ## Liveness probe + ## + livenessProbe: + enabled: true + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + + ## Readiness probe + ## + readinessProbe: + enabled: true + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + + ## Resource limits & requests + ## + resources: {} + # limits: + # cpu: 200m + # memory: 200Mi + # requests: + # cpu: 100m + # memory: 100Mi + + # Required for use in managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), + # because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working + ## + hostNetwork: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## Assign custom affinity rules to the prometheus operator + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + dnsConfig: {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots + # value: "2" + # - name: edns0 + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + + ## Container-specific security context configuration + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + containerSecurityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + ## If false then the user will opt out of automounting API credentials. + ## + automountServiceAccountToken: true + + patch: + enabled: true + image: + repository: rancher/mirrored-ingress-nginx-kube-webhook-certgen + tag: v20221220-controller-v1.5.1-58-g787ea74b6 + sha: "" + pullPolicy: IfNotPresent + resources: {} + ## Provide a priority class name to the webhook patching job + ## + priorityClassName: "" + annotations: {} + # argocd.argoproj.io/hook: PreSync + # argocd.argoproj.io/hook-delete-policy: HookSucceeded + podAnnotations: {} + nodeSelector: {} + affinity: {} + tolerations: [] + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 2000 and gid 2000. *v1.PodSecurityContext false + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 2000 + seccompProfile: + type: RuntimeDefault + + # Security context for create job container + createSecretJob: + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + # Security context for patch job container + patchWebhookJob: + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + # Use certmanager to generate webhook certs + certManager: + enabled: false + # self-signed root certificate + rootCert: + duration: "" # default to be 5y + admissionCert: + duration: "" # default to be 1y + # issuerRef: + # name: "issuer" + # kind: "ClusterIssuer" + + ## Namespaces to scope the interaction of the Prometheus Operator and the apiserver (allow list). + ## This is mutually exclusive with denyNamespaces. Setting this to an empty object will disable the configuration + ## + namespaces: {} + # releaseNamespace: true + # additional: + # - kube-system + + ## Namespaces not to scope the interaction of the Prometheus Operator (deny list). + ## + denyNamespaces: [] + + ## Filter namespaces to look for prometheus-operator custom resources + ## + alertmanagerInstanceNamespaces: [] + alertmanagerConfigNamespaces: [] + prometheusInstanceNamespaces: [] + thanosRulerInstanceNamespaces: [] + + ## The clusterDomain value will be added to the cluster.peer option of the alertmanager. + ## Without this specified option cluster.peer will have value alertmanager-monitoring-alertmanager-0.alertmanager-operated:9094 (default value) + ## With this specified option cluster.peer will have value alertmanager-monitoring-alertmanager-0.alertmanager-operated.namespace.svc.cluster-domain:9094 + ## + # clusterDomain: "cluster.local" + + networkPolicy: + ## Enable creation of NetworkPolicy resources. + ## + enabled: false + + ## Flavor of the network policy to use. + # Can be: + # * kubernetes for networking.k8s.io/v1/NetworkPolicy + # * cilium for cilium.io/v2/CiliumNetworkPolicy + flavor: kubernetes + + # cilium: + # egress: + + ## match labels used in selector + # matchLabels: {} + + ## Service account for Prometheus Operator to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + automountServiceAccountToken: true + + ## Configuration for Prometheus operator service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30080 + + nodePortTls: 30443 + + ## Additional ports to open for Prometheus operator service + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#multi-port-services + ## + additionalPorts: [] + + ## Loadbalancer IP + ## Only use if service.type is "LoadBalancer" + ## + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## NodePort, ClusterIP, LoadBalancer + ## + type: ClusterIP + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + # ## Labels to add to the operator deployment + # ## + labels: {} + + ## Annotations to add to the operator deployment + ## + annotations: {} + + ## Labels to add to the operator pod + ## + podLabels: {} + + ## Annotations to add to the operator pod + ## + podAnnotations: {} + + ## Assign a PriorityClassName to pods if set + # priorityClassName: "" + + ## Define Log Format + # Use logfmt (default) or json logging + # logFormat: logfmt + + ## Decrease log verbosity to errors only + # logLevel: error + + kubeletService: + ## If true, the operator will create and maintain a service for scraping kubelets + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/helm/prometheus-operator/README.md + ## + enabled: true + namespace: kube-system + ## Use '{{ template "kube-prometheus-stack.fullname" . }}-kubelet' by default + name: "" + + ## Create a servicemonitor for the operator + ## + serviceMonitor: + ## If true, create a serviceMonitor for prometheus operator + ## + selfMonitor: true + + ## Labels for ServiceMonitor + additionalLabels: {} + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## Scrape timeout. If not set, the Prometheus default scrape timeout is used. + scrapeTimeout: "" + + ## Metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Resource limits & requests + ## + resources: + limits: + cpu: 200m + memory: 500Mi + requests: + cpu: 100m + memory: 100Mi + + ## Operator Environment + ## env: + ## VARIABLE: value + env: + GOGC: "30" + + # Required for use in managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), + # because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working + ## + hostNetwork: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## Assign custom affinity rules to the prometheus operator + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + dnsConfig: {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots + # value: "2" + # - name: edns0 + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + + ## Container-specific security context configuration + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + containerSecurityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + # Enable vertical pod autoscaler support for prometheus-operator + verticalPodAutoscaler: + enabled: false + + # Recommender responsible for generating recommendation for the object. + # List should be empty (then the default recommender will generate the recommendation) + # or contain exactly one recommender. + # recommenders: + # - name: custom-recommender-performance + + # List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory + controlledResources: [] + # Specifies which resource values should be controlled: RequestsOnly or RequestsAndLimits. + # controlledValues: RequestsAndLimits + + # Define the max allowed resources for the pod + maxAllowed: {} + # cpu: 200m + # memory: 100Mi + # Define the min allowed resources for the pod + minAllowed: {} + # cpu: 200m + # memory: 100Mi + + updatePolicy: + # Specifies minimal number of replicas which need to be alive for VPA Updater to attempt pod eviction + # minReplicas: 1 + # Specifies whether recommended updates are applied when a Pod is started and whether recommended updates + # are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto". + updateMode: Auto + + ## Prometheus-operator image + ## + image: + repository: rancher/mirrored-prometheus-operator-prometheus-operator + tag: v0.72.0 + sha: "" + pullPolicy: IfNotPresent + + ## Prometheus image to use for prometheuses managed by the operator + ## + # prometheusDefaultBaseImage: prometheus/prometheus + + ## Prometheus image registry to use for prometheuses managed by the operator + ## + # prometheusDefaultBaseImageRegistry: quay.io + + ## Alertmanager image to use for alertmanagers managed by the operator + ## + # alertmanagerDefaultBaseImage: prometheus/alertmanager + + ## Alertmanager image registry to use for alertmanagers managed by the operator + ## + # alertmanagerDefaultBaseImageRegistry: quay.io + + ## Prometheus-config-reloader + ## + prometheusConfigReloader: + image: + repository: rancher/mirrored-prometheus-operator-prometheus-config-reloader + tag: v0.72.0 + sha: "" + + # add prometheus config reloader liveness and readiness probe. Default: false + enableProbe: false + + # resource config for prometheusConfigReloader + resources: {} + # requests: + # cpu: 200m + # memory: 50Mi + # limits: + # cpu: 200m + # memory: 50Mi + + ## Thanos side-car image when configured + ## + thanosImage: + repository: rancher/mirrored-thanos-thanos + tag: v0.34.1 + sha: "" + + ## Set a Label Selector to filter watched prometheus and prometheusAgent + ## + prometheusInstanceSelector: "" + + ## Set a Label Selector to filter watched alertmanager + ## + alertmanagerInstanceSelector: "" + + ## Set a Label Selector to filter watched thanosRuler + thanosRulerInstanceSelector: "" + + ## Set a Field Selector to filter watched secrets + ## + secretFieldSelector: "type!=kubernetes.io/dockercfg,type!=kubernetes.io/service-account-token,type!=helm.sh/release.v1" + + ## If false then the user will opt out of automounting API credentials. + ## + automountServiceAccountToken: true + + ## Additional volumes + ## + extraVolumes: [] + + ## Additional volume mounts + ## + extraVolumeMounts: [] + +## Deploy a Prometheus instance +## +prometheus: + enabled: true + + ## Toggle prometheus into agent mode + ## Note many of features described below (e.g. rules, query, alerting, remote read, thanos) will not work in agent mode. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/designs/prometheus-agent.md + ## + agentMode: false + + ## Annotations for Prometheus + ## + annotations: {} + + ## Configure network policy for the prometheus + networkPolicy: + enabled: false + + ## Flavor of the network policy to use. + # Can be: + # * kubernetes for networking.k8s.io/v1/NetworkPolicy + # * cilium for cilium.io/v2/CiliumNetworkPolicy + flavor: kubernetes + + # cilium: + # endpointSelector: + # egress: + # ingress: + + # egress: + # - {} + # ingress: + # - {} + # podSelector: + # matchLabels: + # app: prometheus + + ## Service account for Prometheuses to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + annotations: {} + automountServiceAccountToken: true + + # Service for thanos service discovery on sidecar + # Enable this can make Thanos Query can use + # `--store=dnssrv+_grpc._tcp.${kube-prometheus-stack.fullname}-thanos-discovery.${namespace}.svc.cluster.local` to discovery + # Thanos sidecar on prometheus nodes + # (Please remember to change ${kube-prometheus-stack.fullname} and ${namespace}. Not just copy and paste!) + thanosService: + enabled: false + annotations: {} + labels: {} + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## gRPC port config + portName: grpc + port: 10901 + targetPort: "grpc" + + ## HTTP port config (for metrics) + httpPortName: http + httpPort: 10902 + targetHttpPort: "http" + + ## ClusterIP to assign + # Default is to make this a headless service ("None") + clusterIP: "None" + + ## Port to expose on each node, if service type is NodePort + ## + nodePort: 30901 + httpNodePort: 30902 + + # ServiceMonitor to scrape Sidecar metrics + # Needs thanosService to be enabled as well + thanosServiceMonitor: + enabled: false + interval: "" + + ## Additional labels + ## + additionalLabels: {} + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## Metric relabel configs to apply to samples before ingestion. + metricRelabelings: [] + + ## relabel configs to apply to samples before ingestion. + relabelings: [] + + # Service for external access to sidecar + # Enabling this creates a service to expose thanos-sidecar outside the cluster. + thanosServiceExternal: + enabled: false + annotations: {} + labels: {} + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## gRPC port config + portName: grpc + port: 10901 + targetPort: "grpc" + + ## HTTP port config (for metrics) + httpPortName: http + httpPort: 10902 + targetHttpPort: "http" + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: LoadBalancer + + ## Port to expose on each node + ## + nodePort: 30901 + httpNodePort: 30902 + + ## Configuration for Prometheus service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port for Prometheus Service to listen on + ## + port: 9090 + + ## To be used with a proxy extraContainer port + targetPort: 8081 + + ## Port for Prometheus Reloader to listen on + ## + reloaderWebPort: 8080 + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30090 + + ## Loadbalancer IP + ## Only use if service.type is "LoadBalancer" + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Additional ports to open for Prometheus service + ## + additionalPorts: [] + # additionalPorts: + # - name: oauth-proxy + # port: 8081 + # targetPort: 8081 + # - name: oauth-metrics + # port: 8082 + # targetPort: 8082 + + ## Consider that all endpoints are considered "ready" even if the Pods themselves are not + ## Ref: https://kubernetes.io/docs/reference/kubernetes-api/service-resources/service-v1/#ServiceSpec + publishNotReadyAddresses: false + + ## If you want to make sure that connections from a particular client are passed to the same Pod each time + ## Accepts 'ClientIP' or 'None' + ## + sessionAffinity: None + + ## If you want to modify the ClientIP sessionAffinity timeout + ## The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP" + ## + sessionAffinityConfig: + clientIP: + timeoutSeconds: 10800 + + ## Configuration for creating a separate Service for each statefulset Prometheus replica + ## + servicePerReplica: + enabled: false + annotations: {} + + ## Port for Prometheus Service per replica to listen on + ## + port: 9090 + + ## To be used with a proxy extraContainer port + targetPort: 9090 + + ## Port to expose on each node + ## Only used if servicePerReplica.type is 'NodePort' + ## + nodePort: 30091 + + ## Loadbalancer source IP ranges + ## Only used if servicePerReplica.type is "LoadBalancer" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Configure pod disruption budgets for Prometheus + ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget + ## This configuration is immutable once created and will require the PDB to be deleted to be changed + ## https://github.com/kubernetes/kubernetes/issues/45398 + ## + podDisruptionBudget: + enabled: false + minAvailable: 1 + maxUnavailable: "" + + # Ingress exposes thanos sidecar outside the cluster + thanosIngress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + servicePort: 10901 + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30901 + + ## Hosts must be provided if Ingress is enabled. + ## + hosts: [] + # - thanos-gateway.domain.com + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Thanos Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: thanos-gateway-tls + # hosts: + # - thanos-gateway.domain.com + # + + ## ExtraSecret can be used to store various data in an extra secret + ## (use it for example to store hashed basic auth credentials) + extraSecret: + ## if not set, name will be auto generated + # name: "" + annotations: {} + data: {} + # auth: | + # foo:$apr1$OFG3Xybp$ckL0FHDAkoXYIlH9.cysT0 + # someoneelse:$apr1$DMZX2Z4q$6SbQIfyuLQd.xmo/P0m2c. + + ingress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + + ## Redirect ingress to an additional defined port on the service + # servicePort: 8081 + + ## Hostnames. + ## Must be provided if Ingress is enabled. + ## + # hosts: + # - prometheus.domain.com + hosts: [] + + ## Paths to use for ingress rules - one path should match the prometheusSpec.routePrefix + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Prometheus Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: prometheus-general-tls + # hosts: + # - prometheus.example.com + + ## Configuration for creating an Ingress that will map to each Prometheus replica service + ## prometheus.servicePerReplica must be enabled + ## + ingressPerReplica: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + + ## Final form of the hostname for each per replica ingress is + ## {{ ingressPerReplica.hostPrefix }}-{{ $replicaNumber }}.{{ ingressPerReplica.hostDomain }} + ## + ## Prefix for the per replica ingress that will have `-$replicaNumber` + ## appended to the end + hostPrefix: "" + ## Domain that will be used for the per replica ingress + hostDomain: "" + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## Secret name containing the TLS certificate for Prometheus per replica ingress + ## Secret must be manually created in the namespace + tlsSecretName: "" + + ## Separated secret for each per replica Ingress. Can be used together with cert-manager + ## + tlsSecretPerReplica: + enabled: false + ## Final form of the secret for each per replica ingress is + ## {{ tlsSecretPerReplica.prefix }}-{{ $replicaNumber }} + ## + prefix: "prometheus" + + ## Configure additional options for default pod security policy for Prometheus + ## ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ + podSecurityPolicy: + allowedCapabilities: [] + allowedHostPaths: [] + volumes: [] + + serviceMonitor: + ## If true, create a serviceMonitor for prometheus + ## + selfMonitor: true + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## Additional labels + ## + additionalLabels: {} + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## Metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional Endpoints + ## + additionalEndpoints: [] + # - port: oauth-metrics + # path: /metrics + + ## Settings affecting prometheusSpec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#prometheusspec + ## + prometheusSpec: + ## If true, pass --storage.tsdb.max-block-duration=2h to prometheus. This is already done if using Thanos + ## + disableCompaction: false + ## APIServerConfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#apiserverconfig + ## + apiserverConfig: {} + + ## Allows setting additional arguments for the Prometheus container + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.Prometheus + additionalArgs: [] + + ## Interval between consecutive scrapes. + ## Defaults to 30s. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/release-0.44/pkg/prometheus/promcfg.go#L180-L183 + ## + scrapeInterval: "30s" + + ## Number of seconds to wait for target to respond before erroring + ## + # scrapeTimeout: "30s" + + ## Interval between consecutive evaluations. + ## + evaluationInterval: "30s" + + ## ListenLocal makes the Prometheus server listen on loopback, so that it does not bind against the Pod IP. + ## + listenLocal: false + + ## EnableAdminAPI enables Prometheus the administrative HTTP API which includes functionality such as deleting time series. + ## This is disabled by default. + ## ref: https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-admin-apis + ## + enableAdminAPI: false + + ## Sets version of Prometheus overriding the Prometheus version as derived + ## from the image tag. Useful in cases where the tag does not follow semver v2. + version: "" + + ## WebTLSConfig defines the TLS parameters for HTTPS + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#webtlsconfig + web: {} + + ## Exemplars related settings that are runtime reloadable. + ## It requires to enable the exemplar storage feature to be effective. + exemplars: "" + ## Maximum number of exemplars stored in memory for all series. + ## If not set, Prometheus uses its default value. + ## A value of zero or less than zero disables the storage. + # maxSize: 100000 + + # EnableFeatures API enables access to Prometheus disabled features. + # ref: https://prometheus.io/docs/prometheus/latest/disabled_features/ + enableFeatures: [] + # - exemplar-storage + + ## Image of Prometheus. + ## + image: + repository: rancher/mirrored-prometheus-prometheus + tag: v2.50.1 + sha: "" + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## If specified, the pod's topology spread constraints. + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app: prometheus + + ## Alertmanagers to which alerts will be sent + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerendpoints + ## + ## Default configuration will connect to the alertmanager deployed as part of this release + ## + alertingEndpoints: [] + # - name: "" + # namespace: "" + # port: http + # scheme: http + # pathPrefix: "" + # tlsConfig: {} + # bearerTokenFile: "" + # apiVersion: v2 + + ## External labels to add to any time series or alerts when communicating with external systems + ## + externalLabels: {} + + ## enable --web.enable-remote-write-receiver flag on prometheus-server + ## + enableRemoteWriteReceiver: false + + ## Name of the external label used to denote replica name + ## + replicaExternalLabelName: "" + + ## If true, the Operator won't add the external label used to denote replica name + ## + replicaExternalLabelNameClear: false + + ## Name of the external label used to denote Prometheus instance name + ## + prometheusExternalLabelName: "" + + ## If true, the Operator won't add the external label used to denote Prometheus instance name + ## + prometheusExternalLabelNameClear: false + + ## External URL at which Prometheus will be reachable. + ## + externalUrl: "" + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Secrets is a list of Secrets in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. + ## The Secrets are mounted into /etc/prometheus/secrets/. Secrets changes after initial creation of a Prometheus object are not + ## reflected in the running Pods. To change the secrets mounted into the Prometheus Pods, the object must be deleted and recreated + ## with the new list of secrets. + ## + secrets: [] + + ## ConfigMaps is a list of ConfigMaps in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. + ## The ConfigMaps are mounted into /etc/prometheus/configmaps/. + ## + configMaps: [] + + ## QuerySpec defines the query command line flags when starting Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#queryspec + ## + query: {} + + ## If nil, select own namespace. Namespaces to be selected for PrometheusRules discovery. + ruleNamespaceSelector: {} + ## Example which selects PrometheusRules in namespaces with label "prometheus" set to "somelabel" + # ruleNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.ruleSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the PrometheusRule resources created + ## + ruleSelectorNilUsesHelmValues: false + + ## PrometheusRules to be selected for target discovery. + ## If {}, select all PrometheusRules + ## + ruleSelector: {} + ## Example which select all PrometheusRules resources + ## with label "prometheus" with values any of "example-rules" or "example-rules-2" + # ruleSelector: + # matchExpressions: + # - key: prometheus + # operator: In + # values: + # - example-rules + # - example-rules-2 + # + ## Example which select all PrometheusRules resources with label "role" set to "example-rules" + # ruleSelector: + # matchLabels: + # role: example-rules + + ## If true, a nil or {} value for prometheus.prometheusSpec.serviceMonitorSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the servicemonitors created + ## + serviceMonitorSelectorNilUsesHelmValues: false + + ## ServiceMonitors to be selected for target discovery. + ## If {}, select all ServiceMonitors + ## + serviceMonitorSelector: {} + ## Example which selects ServiceMonitors with label "prometheus" set to "somelabel" + # serviceMonitorSelector: + # matchLabels: + # prometheus: somelabel + + ## Namespaces to be selected for ServiceMonitor discovery. + ## + serviceMonitorNamespaceSelector: {} + ## Example which selects ServiceMonitors in namespaces with label "prometheus" set to "somelabel" + # serviceMonitorNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.podMonitorSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the podmonitors created + ## + podMonitorSelectorNilUsesHelmValues: false + + ## PodMonitors to be selected for target discovery. + ## If {}, select all PodMonitors + ## + podMonitorSelector: {} + ## Example which selects PodMonitors with label "prometheus" set to "somelabel" + # podMonitorSelector: + # matchLabels: + # prometheus: somelabel + + ## If nil, select own namespace. Namespaces to be selected for PodMonitor discovery. + podMonitorNamespaceSelector: {} + ## Example which selects PodMonitor in namespaces with label "prometheus" set to "somelabel" + # podMonitorNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.probeSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the probes created + ## + probeSelectorNilUsesHelmValues: true + + ## Probes to be selected for target discovery. + ## If {}, select all Probes + ## + probeSelector: {} + ## Example which selects Probes with label "prometheus" set to "somelabel" + # probeSelector: + # matchLabels: + # prometheus: somelabel + + ## If nil, select own namespace. Namespaces to be selected for Probe discovery. + probeNamespaceSelector: {} + ## Example which selects Probe in namespaces with label "prometheus" set to "somelabel" + # probeNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.scrapeConfigSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the scrapeConfigs created + ## + scrapeConfigSelectorNilUsesHelmValues: true + + ## scrapeConfigs to be selected for target discovery. + ## If {}, select all scrapeConfigs + ## + scrapeConfigSelector: {} + ## Example which selects scrapeConfigs with label "prometheus" set to "somelabel" + # scrapeConfigSelector: + # matchLabels: + # prometheus: somelabel + + ## If nil, select own namespace. Namespaces to be selected for scrapeConfig discovery. + scrapeConfigNamespaceSelector: {} + ## Example which selects scrapeConfig in namespaces with label "prometheus" set to "somelabel" + # scrapeConfigNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## How long to retain metrics + ## + retention: 10d + + ## Maximum size of metrics + ## + retentionSize: "" + + ## Allow out-of-order/out-of-bounds samples ingested into Prometheus for a specified duration + ## See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tsdb + tsdb: + outOfOrderTimeWindow: 0s + + ## Enable compression of the write-ahead log using Snappy. + ## + walCompression: true + + ## If true, the Operator won't process any Prometheus configuration changes + ## + paused: false + + ## Number of replicas of each shard to deploy for a Prometheus deployment. + ## Number of replicas multiplied by shards is the total number of Pods created. + ## + replicas: 1 + + ## EXPERIMENTAL: Number of shards to distribute targets onto. + ## Number of replicas multiplied by shards is the total number of Pods created. + ## Note that scaling down shards will not reshard data onto remaining instances, it must be manually moved. + ## Increasing shards will not reshard data either but it will continue to be available from the same instances. + ## To query globally use Thanos sidecar and Thanos querier or remote write data to a central location. + ## Sharding is done on the content of the `__address__` target meta-label. + ## + shards: 1 + + ## Log level for Prometheus be configured in + ## + logLevel: info + + ## Log format for Prometheus be configured in + ## + logFormat: logfmt + + ## Prefix used to register routes, overriding externalUrl route. + ## Useful for proxies that rewrite URLs. + ## + routePrefix: / + + ## Standard object's metadata. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata + ## Metadata Labels and Annotations gets propagated to the prometheus pods. + ## + podMetadata: {} + # labels: + # app: prometheus + # k8s-app: prometheus + + ## Pod anti-affinity can prevent the scheduler from placing Prometheus replicas on the same node. + ## The default value "soft" means that the scheduler should *prefer* to not schedule two replica pods onto the same node but no guarantee is provided. + ## The value "hard" means that the scheduler is *required* to not schedule two replica pods onto the same node. + ## The value "" will disable pod anti-affinity so that no anti-affinity rules will be configured. + podAntiAffinity: "" + + ## If anti-affinity is enabled sets the topologyKey to use for anti-affinity. + ## This can be changed to, for example, failure-domain.beta.kubernetes.io/zone + ## + podAntiAffinityTopologyKey: kubernetes.io/hostname + + ## Assign custom affinity rules to the prometheus instance + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + + ## The remote_read spec configuration for Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#remotereadspec + remoteRead: [] + # - url: http://remote1/read + ## additionalRemoteRead is appended to remoteRead + additionalRemoteRead: [] + + ## The remote_write spec configuration for Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#remotewritespec + remoteWrite: [] + # - url: http://remote1/push + ## additionalRemoteWrite is appended to remoteWrite + additionalRemoteWrite: [] + + ## Enable/Disable Grafana dashboards provisioning for prometheus remote write feature + remoteWriteDashboards: false + + ## Resource limits & requests + ## + resources: + limits: + memory: 3000Mi + cpu: 1000m + requests: + memory: 750Mi + cpu: 750m + + ## Prometheus StorageSpec for persistent data + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/storage.md + ## + storageSpec: {} + ## Using PersistentVolumeClaim + ## + # volumeClaimTemplate: + # spec: + # storageClassName: gluster + # accessModes: ["ReadWriteOnce"] + # resources: + # requests: + # storage: 50Gi + # selector: {} + + ## Using tmpfs volume + ## + # emptyDir: + # medium: Memory + + # Additional volumes on the output StatefulSet definition. + volumes: + - name: nginx-home + emptyDir: {} + - name: prometheus-nginx + configMap: + name: prometheus-nginx-proxy-config + defaultMode: 438 + + # Additional VolumeMounts on the output StatefulSet definition. + volumeMounts: [] + + ## AdditionalScrapeConfigs allows specifying additional Prometheus scrape configurations. Scrape configurations + ## are appended to the configurations generated by the Prometheus Operator. Job configurations must have the form + ## as specified in the official Prometheus documentation: + ## https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config. As scrape configs are + ## appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility + ## to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible + ## scrape configs are going to break Prometheus after the upgrade. + ## AdditionalScrapeConfigs can be defined as a list or as a templated string. + ## + ## The scrape configuration example below will find master nodes, provided they have the name .*mst.*, relabel the + ## port to 2379 and allow etcd scraping provided it is running on all Kubernetes master nodes + ## + additionalScrapeConfigs: [] + # - job_name: kube-etcd + # kubernetes_sd_configs: + # - role: node + # scheme: https + # tls_config: + # ca_file: /etc/prometheus/secrets/etcd-client-cert/etcd-ca + # cert_file: /etc/prometheus/secrets/etcd-client-cert/etcd-client + # key_file: /etc/prometheus/secrets/etcd-client-cert/etcd-client-key + # relabel_configs: + # - action: labelmap + # regex: __meta_kubernetes_node_label_(.+) + # - source_labels: [__address__] + # action: replace + # targetLabel: __address__ + # regex: ([^:;]+):(\d+) + # replacement: ${1}:2379 + # - source_labels: [__meta_kubernetes_node_name] + # action: keep + # regex: .*mst.* + # - source_labels: [__meta_kubernetes_node_name] + # action: replace + # targetLabel: node + # regex: (.*) + # replacement: ${1} + # metric_relabel_configs: + # - regex: (kubernetes_io_hostname|failure_domain_beta_kubernetes_io_region|beta_kubernetes_io_os|beta_kubernetes_io_arch|beta_kubernetes_io_instance_type|failure_domain_beta_kubernetes_io_zone) + # action: labeldrop + # + ## If scrape config contains a repetitive section, you may want to use a template. + ## In the following example, you can see how to define `gce_sd_configs` for multiple zones + # additionalScrapeConfigs: | + # - job_name: "node-exporter" + # gce_sd_configs: + # {{range $zone := .Values.gcp_zones}} + # - project: "project1" + # zone: "{{$zone}}" + # port: 9100 + # {{end}} + # relabel_configs: + # ... + + + ## If additional scrape configurations are already deployed in a single secret file you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalScrapeConfigs + additionalScrapeConfigsSecret: {} + # enabled: false + # name: + # key: + + ## additionalPrometheusSecretsAnnotations allows to add annotations to the kubernetes secret. This can be useful + ## when deploying via spinnaker to disable versioning on the secret, strategy.spinnaker.io/versioned: 'false' + additionalPrometheusSecretsAnnotations: {} + + ## AdditionalAlertManagerConfigs allows for manual configuration of alertmanager jobs in the form as specified + ## in the official Prometheus documentation https://prometheus.io/docs/prometheus/latest/configuration/configuration/#. + ## AlertManager configurations specified are appended to the configurations generated by the Prometheus Operator. + ## As AlertManager configs are appended, the user is responsible to make sure it is valid. Note that using this + ## feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release + ## notes to ensure that no incompatible AlertManager configs are going to break Prometheus after the upgrade. + ## + additionalAlertManagerConfigs: [] + # - consul_sd_configs: + # - server: consul.dev.test:8500 + # scheme: http + # datacenter: dev + # tag_separator: ',' + # services: + # - metrics-prometheus-alertmanager + + ## If additional alertmanager configurations are already deployed in a single secret, or you want to manage + ## them separately from the helm deployment, you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalAlertManagerConfigs + additionalAlertManagerConfigsSecret: {} + # name: + # key: + # optional: false + + ## AdditionalAlertRelabelConfigs allows specifying Prometheus alert relabel configurations. Alert relabel configurations specified are appended + ## to the configurations generated by the Prometheus Operator. Alert relabel configurations specified must have the form as specified in the + ## official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alert_relabel_configs. + ## As alert relabel configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the + ## possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible alert relabel + ## configs are going to break Prometheus after the upgrade. + ## + additionalAlertRelabelConfigs: [] + # - separator: ; + # regex: prometheus_replica + # replacement: $1 + # action: labeldrop + + ## If additional alert relabel configurations are already deployed in a single secret, or you want to manage + ## them separately from the helm deployment, you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalAlertRelabelConfigs + additionalAlertRelabelConfigsSecret: {} + # name: + # key: + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 1000 and gid 2000. + ## https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + + ## Priority class assigned to the Pods + ## + priorityClassName: "" + + ## Thanos configuration allows configuring various aspects of a Prometheus server in a Thanos environment. + ## This section is experimental, it may change significantly without deprecation notice in any release. + ## This is experimental and may change significantly without backward compatibility in any release. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#thanosspec + ## + thanos: {} + # secretProviderClass: + # provider: gcp + # parameters: + # secrets: | + # - resourceName: "projects/$PROJECT_ID/secrets/testsecret/versions/latest" + # fileName: "objstore.yaml" + ## ObjectStorageConfig configures object storage in Thanos. + # objectStorageConfig: + # # use existing secret, if configured, objectStorageConfig.secret will not be used + # existingSecret: {} + # # name: "" + # # key: "" + # # will render objectStorageConfig secret data and configure it to be used by Thanos custom resource, + # # ignored when prometheusspec.thanos.objectStorageConfig.existingSecret is set + # # https://thanos.io/tip/thanos/storage.md/#s3 + # secret: {} + # # type: S3 + # # config: + # # bucket: "" + # # endpoint: "" + # # region: "" + # # access_key: "" + # # secret_key: "" + + proxy: + image: + repository: rancher/mirrored-library-nginx + tag: 1.24.0-alpine + + ## Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to a Prometheus pod. + ## if using proxy extraContainer update targetPort with proxy container port + containers: | + - name: prometheus-proxy + args: + - nginx + - -g + - daemon off; + - -c + - /nginx/nginx.conf + image: "{{ template "system_default_registry" . }}{{ .Values.prometheus.prometheusSpec.proxy.image.repository }}:{{ .Values.prometheus.prometheusSpec.proxy.image.tag }}" + ports: + - containerPort: 8081 + name: nginx-http + protocol: TCP + volumeMounts: + - mountPath: /nginx + name: prometheus-nginx + - mountPath: /var/cache/nginx + name: nginx-home + securityContext: + runAsUser: 101 + runAsGroup: 101 + + ## InitContainers allows injecting additional initContainers. This is meant to allow doing some changes + ## (permissions, dir tree) on mounted volumes before starting prometheus + initContainers: [] + + ## PortName to use for Prometheus. + ## + portName: "http-web" + + ## ArbitraryFSAccessThroughSMs configures whether configuration based on a service monitor can access arbitrary files + ## on the file system of the Prometheus container e.g. bearer token files. + arbitraryFSAccessThroughSMs: false + + ## OverrideHonorLabels if set to true overrides all user configured honor_labels. If HonorLabels is set in ServiceMonitor + ## or PodMonitor to true, this overrides honor_labels to false. + overrideHonorLabels: false + + ## OverrideHonorTimestamps allows to globally enforce honoring timestamps in all scrape configs. + overrideHonorTimestamps: false + + ## When ignoreNamespaceSelectors is set to true, namespaceSelector from all PodMonitor, ServiceMonitor and Probe objects will be ignored, + ## they will only discover targets within the namespace of the PodMonitor, ServiceMonitor and Probe object, + ## and servicemonitors will be installed in the default service namespace. + ## Defaults to false. + ignoreNamespaceSelectors: true + + ## EnforcedNamespaceLabel enforces adding a namespace label of origin for each alert and metric that is user created. + ## The label value will always be the namespace of the object that is being created. + ## Disabled by default + enforcedNamespaceLabel: "" + + ## PrometheusRulesExcludedFromEnforce - list of prometheus rules to be excluded from enforcing of adding namespace labels. + ## Works only if enforcedNamespaceLabel set to true. Make sure both ruleNamespace and ruleName are set for each pair + ## Deprecated, use `excludedFromEnforcement` instead + prometheusRulesExcludedFromEnforce: [] + + ## ExcludedFromEnforcement - list of object references to PodMonitor, ServiceMonitor, Probe and PrometheusRule objects + ## to be excluded from enforcing a namespace label of origin. + ## Works only if enforcedNamespaceLabel set to true. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#objectreference + excludedFromEnforcement: [] + + ## QueryLogFile specifies the file to which PromQL queries are logged. Note that this location must be writable, + ## and can be persisted using an attached volume. Alternatively, the location can be set to a stdout location such + ## as /dev/stdout to log querie information to the default Prometheus log stream. This is only available in versions + ## of Prometheus >= 2.16.0. For more details, see the Prometheus docs (https://prometheus.io/docs/guides/query-log/) + queryLogFile: false + + # Use to set global sample_limit for Prometheus. This act as default SampleLimit for ServiceMonitor or/and PodMonitor. + # Set to 'false' to disable global sample_limit. or set to a number to override the default value. + sampleLimit: false + + # EnforcedKeepDroppedTargetsLimit defines on the number of targets dropped by relabeling that will be kept in memory. + # The value overrides any spec.keepDroppedTargets set by ServiceMonitor, PodMonitor, Probe objects unless spec.keepDroppedTargets + # is greater than zero and less than spec.enforcedKeepDroppedTargets. 0 means no limit. + enforcedKeepDroppedTargets: 0 + + ## EnforcedSampleLimit defines global limit on number of scraped samples that will be accepted. This overrides any SampleLimit + ## set per ServiceMonitor or/and PodMonitor. It is meant to be used by admins to enforce the SampleLimit to keep overall + ## number of samples/series under the desired limit. Note that if SampleLimit is lower that value will be taken instead. + enforcedSampleLimit: false + + ## EnforcedTargetLimit defines a global limit on the number of scraped targets. This overrides any TargetLimit set + ## per ServiceMonitor or/and PodMonitor. It is meant to be used by admins to enforce the TargetLimit to keep the overall + ## number of targets under the desired limit. Note that if TargetLimit is lower, that value will be taken instead, except + ## if either value is zero, in which case the non-zero value will be used. If both values are zero, no limit is enforced. + enforcedTargetLimit: false + + + ## Per-scrape limit on number of labels that will be accepted for a sample. If more than this number of labels are present + ## post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus versions + ## 2.27.0 and newer. + enforcedLabelLimit: false + + ## Per-scrape limit on length of labels name that will be accepted for a sample. If a label name is longer than this number + ## post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus versions + ## 2.27.0 and newer. + enforcedLabelNameLengthLimit: false + + ## Per-scrape limit on length of labels value that will be accepted for a sample. If a label value is longer than this + ## number post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus + ## versions 2.27.0 and newer. + enforcedLabelValueLengthLimit: false + + ## AllowOverlappingBlocks enables vertical compaction and vertical query merge in Prometheus. This is still experimental + ## in Prometheus so it may change in any upcoming release. + allowOverlappingBlocks: false + + ## Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to + ## be considered available. Defaults to 0 (pod will be considered available as soon as it is ready). + minReadySeconds: 0 + + # Required for use in managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), + # because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working + # Use the host's network namespace if true. Make sure to understand the security implications if you want to enable it. + # When hostNetwork is enabled, this will set dnsPolicy to ClusterFirstWithHostNet automatically. + hostNetwork: false + + # HostAlias holds the mapping between IP and hostnames that will be injected + # as an entry in the pod’s hosts file. + hostAliases: [] + # - ip: 10.10.0.100 + # hostnames: + # - a1.app.local + # - b1.app.local + + ## TracingConfig configures tracing in Prometheus. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#prometheustracingconfig + tracingConfig: {} + + ## Additional configuration which is not covered by the properties above. (passed through tpl) + additionalConfig: {} + + ## Additional configuration which is not covered by the properties above. + ## Useful, if you need advanced templating inside alertmanagerSpec. + ## Otherwise, use prometheus.prometheusSpec.additionalConfig (passed through tpl) + additionalConfigString: "" + + ## Defines the maximum time that the `prometheus` container's startup probe + ## will wait before being considered failed. The startup probe will return + ## success after the WAL replay is complete. If set, the value should be + ## greater than 60 (seconds). Otherwise it will be equal to 600 seconds (15 + ## minutes). + maximumStartupDurationSeconds: 0 + + additionalRulesForClusterRole: [] + # - apiGroups: [ "" ] + # resources: + # - nodes/proxy + # verbs: [ "get", "list", "watch" ] + + additionalServiceMonitors: [] + ## Name of the ServiceMonitor to create + ## + # - name: "" + + ## Additional labels to set used for the ServiceMonitorSelector. Together with standard labels from + ## the chart + ## + # additionalLabels: {} + + ## Service label for use in assembling a job name of the form